diff options
250 files changed, 24064 insertions, 6396 deletions
diff --git a/.custom-format.py b/.custom-format.py index 1295ce55..c8075ace 100755 --- a/.custom-format.py +++ b/.custom-format.py @@ -24,7 +24,7 @@ import importlib.metadata # string_re = re.compile('(\\"(\\\\.|[^"\\\\])*\\")') # TODO: for future use -CURRENT_LLVM = os.getenv('LLVM_VERSION', 14) +CURRENT_LLVM = os.getenv('LLVM_VERSION', 17) CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN", "") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdf618b9..ed382fbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,20 +20,18 @@ jobs: AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 steps: - uses: actions/checkout@v3 + - name: update + run: sudo apt-get update && sudo apt-get upgrade -y - name: debug run: apt-cache search plugin-dev | grep gcc-; echo; apt-cache search clang-format- | grep clang-format- - - name: update - run: sudo apt-get update - # && sudo apt-get upgrade -y - name: install packages - #run: sudo apt-get install -y -m -f --install-suggests build-essential git libtool libtool-bin automake bison libglib2.0-0 clang llvm-dev libc++-dev findutils libcmocka-dev python3-dev python3-setuptools ninja-build python3-pip - run: sudo apt-get install -y -m -f build-essential git libtool libtool-bin automake flex bison libglib2.0-0 clang llvm-dev libc++-dev findutils libcmocka-dev python3-dev python3-setuptools ninja-build python3-pip + run: sudo apt-get install -y -m -f build-essential gcc-10 g++-10 git libtool libtool-bin automake flex bison libglib2.0-0 clang-12 llvm-12-dev libc++-dev findutils libcmocka-dev python3-dev python3-setuptools ninja-build python3-pip gcc-10-plugin-dev - name: compiler installed run: gcc -v; echo; clang -v - name: install gcc plugin run: sudo apt-get install -y -m -f --install-suggests $(readlink /usr/bin/gcc)-plugin-dev - name: build afl++ - run: make distrib ASAN_BUILD=1 NO_NYX=1 + run: export NO_NYX=1; export ASAN_BUILD=1; export LLVM_CONFIG=llvm-config-12; make ASAN_BUILD=1 NO_NYX=1 LLVM_CONFIG=llvm-config-12 distrib - name: run tests run: sudo -E ./afl-system-config; make tests # macos: diff --git a/.gitignore b/.gitignore index c01750e1..8e191e29 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ .test .test2 .vscode +afl-addseeds.8 afl-analyze afl-analyze.8 afl-as @@ -98,10 +99,15 @@ unicorn_mode/samples/*/\.test-* utils/afl_network_proxy/afl-network-client utils/afl_network_proxy/afl-network-server utils/afl_proxy/afl-proxy +utils/bench/hash utils/optimin/build utils/optimin/optimin utils/persistent_mode/persistent_demo utils/persistent_mode/persistent_demo_new +utils/persistent_mode/persistent_demo_new_compat utils/persistent_mode/test-instr +utils/replay_record/persistent_demo_replay +utils/replay_record/persistent_demo_replay_compat +utils/replay_record/persistent_demo_replay_argparse utils/plot_ui/afl-plot-ui vuln_prog diff --git a/.gitmodules b/.gitmodules index 18fda27e..7fce4460 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,9 +19,9 @@ [submodule "nyx_mode/libnyx"] path = nyx_mode/libnyx url = https://github.com/nyx-fuzz/libnyx.git -[submodule "nyx_mode/QEMU-Nyx"] - path = nyx_mode/QEMU-Nyx - url = https://github.com/nyx-fuzz/qemu-nyx.git [submodule "nyx_mode/packer"] path = nyx_mode/packer url = https://github.com/nyx-fuzz/packer.git +[submodule "nyx_mode/QEMU-Nyx"] + path = nyx_mode/QEMU-Nyx + url = https://github.com/nyx-fuzz/QEMU-Nyx diff --git a/CITATION.cff b/CITATION.cff index 37a4a174..5ae7211d 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -27,5 +27,5 @@ keywords: - qemu - llvm - unicorn-emulator - - securiy + - security license: AGPL-3.0-or-later diff --git a/Dockerfile b/Dockerfile index 1b5ffd28..99998a61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ ENV NO_CORESIGHT=1 ENV NO_NYX=1 ### Only change these if you know what you are doing: -# LLVM 15 does not look good so we stay at 14 to still have LTO -ENV LLVM_VERSION=14 +# Current recommended LLVM version is 16 +ENV LLVM_VERSION=16 # GCC 12 is producing compile errors for some targets so we stay at GCC 11 ENV GCC_VERSION=11 @@ -42,7 +42,7 @@ RUN apt-get update && \ python3 python3-dev python3-pip python-is-python3 \ libtool libtool-bin libglib2.0-dev \ apt-transport-https gnupg dialog \ - gnuplot-nox libpixman-1-dev \ + gnuplot-nox libpixman-1-dev bc \ gcc-${GCC_VERSION} g++-${GCC_VERSION} gcc-${GCC_VERSION}-plugin-dev gdb lcov \ clang-${LLVM_VERSION} clang-tools-${LLVM_VERSION} libc++1-${LLVM_VERSION} \ libc++-${LLVM_VERSION}-dev libc++abi1-${LLVM_VERSION} libc++abi-${LLVM_VERSION}-dev \ @@ -88,7 +88,7 @@ ARG TEST_BUILD RUN sed -i.bak 's/^ -/ /g' GNUmakefile && \ make clean && make distrib && \ - ([ "${TEST_BUILD}" ] || (make install && make clean)) && \ + ([ "${TEST_BUILD}" ] || (make install)) && \ mv GNUmakefile.bak GNUmakefile RUN echo "set encoding=utf-8" > /root/.vimrc && \ diff --git a/GNUmakefile b/GNUmakefile index 31374c10..dee9bbb3 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -32,7 +32,7 @@ VERSION = $(shell grep '^$(HASH)define VERSION ' ../config.h | cut -d '"' -f # PROGS intentionally omit afl-as, which gets installed elsewhere. PROGS = afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze -SH_PROGS = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-system-config afl-persistent-config afl-cc +SH_PROGS = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-addseeds afl-system-config afl-persistent-config afl-cc MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) afl-as.8 ASAN_OPTIONS=detect_leaks=0 @@ -45,10 +45,14 @@ ifdef NO_SPLICING override CFLAGS_OPT += -DNO_SPLICING endif +ifdef NO_UTF + override CFLAGS_OPT += -DFANCY_BOXES_NO_UTF +endif + ifdef ASAN_BUILD $(info Compiling ASAN version of binaries) override CFLAGS += $(ASAN_CFLAGS) - LDFLAGS += $(ASAN_LDFLAGS) + override LDFLAGS += $(ASAN_LDFLAGS) endif ifdef UBSAN_BUILD $(info Compiling UBSAN version of binaries) @@ -62,6 +66,10 @@ ifdef MSAN_BUILD override LDFLAGS += -fsanitize=memory endif +ifdef CODE_COVERAGE + override CFLAGS += -D__AFL_CODE_COVERAGE=1 +endif + ifeq "$(findstring android, $(shell $(CC) --version 2>/dev/null))" "" ifeq "$(shell echo 'int main() {return 0; }' | $(CC) $(CFLAGS) -Werror -x c - -flto=full -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" CFLAGS_FLTO ?= -flto=full @@ -76,33 +84,44 @@ else endif endif -#ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -fno-move-loop-invariants -fdisable-tree-cunrolli -x c - -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" -# SPECIAL_PERFORMANCE += -fno-move-loop-invariants -fdisable-tree-cunrolli -#endif - -#ifeq "$(shell echo 'int main() {return 0; }' | $(CC) $(CFLAGS) -Werror -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" -# ifndef SOURCE_DATE_EPOCH -# HAVE_MARCHNATIVE = 1 -# CFLAGS_OPT += -march=native -# endif -#endif +ifdef PERFORMANCE + SPECIAL_PERFORMANCE := -D_AFL_SPECIAL_PERFORMANCE + ifeq "$(SYS)" "Linux" + ifeq "$(shell grep avx2 /proc/cpuinfo)" "" + else + SPECIAL_PERFORMANCE += -mavx2 -D_HAVE_AVX2 + endif + endif + ifeq "$(shell echo 'int main() {return 0; }' | $(CC) $(CFLAGS) -Werror -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + HAVE_MARCHNATIVE = 1 + SPECIAL_PERFORMANCE += -march=native + endif + $(info SPECIAL_PERFORMANCE=$(SPECIAL_PERFORMANCE)) +else + SPECIAL_PERFORMANCE := +endif ifneq "$(SYS)" "Darwin" - #ifeq "$(HAVE_MARCHNATIVE)" "1" - # SPECIAL_PERFORMANCE += -march=native - #endif - ifndef DEBUG - CFLAGS_OPT += -D_FORTIFY_SOURCE=1 - endif + #ifeq "$(HAVE_MARCHNATIVE)" "1" + # SPECIAL_PERFORMANCE += -march=native + #endif + #ifndef DEBUG + # override CFLAGS_OPT += -D_FORTIFY_SOURCE=1 + #endif else # On some odd MacOS system configurations, the Xcode sdk path is not set correctly SDK_LD = -L$(shell xcrun --show-sdk-path)/usr/lib - LDFLAGS += $(SDK_LD) + override LDFLAGS += $(SDK_LD) +endif + +COMPILER_TYPE=$(shell $(CC) --version|grep "Free Software Foundation") +ifneq "$(COMPILER_TYPE)" "" + #$(info gcc is being used) + override CFLAGS_OPT += -Wno-error=format-truncation -Wno-format-truncation endif ifeq "$(SYS)" "SunOS" - CFLAGS_OPT += -Wno-format-truncation - LDFLAGS = -lkstat -lrt -lsocket -lnsl + override LDFLAGS = -lkstat -lrt -lsocket -lnsl endif ifdef STATIC @@ -112,8 +131,8 @@ ifdef STATIC PYFLAGS= PYTHON_INCLUDE = / - CFLAGS_OPT += -static - LDFLAGS += -lm -lpthread -lz -lutil + override CFLAGS_OPT += -static + override LDFLAGS += -lm -lpthread -lz -lutil endif ifdef PROFILING @@ -145,7 +164,7 @@ else endif override CFLAGS += -g -Wno-pointer-sign -Wno-variadic-macros -Wall -Wextra -Wno-pointer-arith \ - -fPIC -I include/ -DAFL_PATH=\"$(HELPER_PATH)\" \ + -fPIC -I include/ -DAFL_PATH=\"$(HELPER_PATH)\" \ -DBIN_PATH=\"$(BIN_PATH)\" -DDOC_PATH=\"$(DOC_PATH)\" # -fstack-protector @@ -180,13 +199,13 @@ AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) ifneq "$(shell command -v python3m 2>/dev/null)" "" ifneq "$(shell command -v python3m-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python3m-config --includes) - PYTHON_VERSION ?= $(strip $(shell python3m --version 2>&1)) + PYTHON_INCLUDE := $(shell python3m-config --includes) + PYTHON_VERSION := $(strip $(shell python3m --version 2>&1)) # Starting with python3.8, we need to pass the `embed` flag. Earlier versions didn't know this flag. ifeq "$(shell python3m-config --embed --libs 2>/dev/null | grep -q lpython && echo 1 )" "1" - PYTHON_LIB ?= $(shell python3m-config --libs --embed --ldflags) + PYTHON_LIB := $(shell python3m-config --libs --embed --ldflags) else - PYTHON_LIB ?= $(shell python3m-config --ldflags) + PYTHON_LIB := $(shell python3m-config --ldflags) endif endif endif @@ -194,13 +213,13 @@ endif ifeq "$(PYTHON_INCLUDE)" "" ifneq "$(shell command -v python3 2>/dev/null)" "" ifneq "$(shell command -v python3-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python3-config --includes) - PYTHON_VERSION ?= $(strip $(shell python3 --version 2>&1)) + PYTHON_INCLUDE := $(shell python3-config --includes) + PYTHON_VERSION := $(strip $(shell python3 --version 2>&1)) # Starting with python3.8, we need to pass the `embed` flag. Earlier versions didn't know this flag. ifeq "$(shell python3-config --embed --libs 2>/dev/null | grep -q lpython && echo 1 )" "1" - PYTHON_LIB ?= $(shell python3-config --libs --embed --ldflags) + PYTHON_LIB := $(shell python3-config --libs --embed --ldflags) else - PYTHON_LIB ?= $(shell python3-config --ldflags) + PYTHON_LIB := $(shell python3-config --ldflags) endif endif endif @@ -209,9 +228,9 @@ endif ifeq "$(PYTHON_INCLUDE)" "" ifneq "$(shell command -v python 2>/dev/null)" "" ifneq "$(shell command -v python-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python-config --includes) - PYTHON_LIB ?= $(shell python-config --ldflags) - PYTHON_VERSION ?= $(strip $(shell python --version 2>&1)) + PYTHON_INCLUDE := $(shell python-config --includes) + PYTHON_LIB := $(shell python-config --ldflags) + PYTHON_VERSION := $(strip $(shell python --version 2>&1)) endif endif endif @@ -220,9 +239,9 @@ endif ifeq "$(PYTHON_INCLUDE)" "" ifneq "$(shell command -v python3.7 2>/dev/null)" "" ifneq "$(shell command -v python3.7-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python3.7-config --includes) - PYTHON_LIB ?= $(shell python3.7-config --ldflags) - PYTHON_VERSION ?= $(strip $(shell python3.7 --version 2>&1)) + PYTHON_INCLUDE := $(shell python3.7-config --includes) + PYTHON_LIB := $(shell python3.7-config --ldflags) + PYTHON_VERSION := $(strip $(shell python3.7 --version 2>&1)) endif endif endif @@ -231,9 +250,9 @@ endif ifeq "$(PYTHON_INCLUDE)" "" ifneq "$(shell command -v python2.7 2>/dev/null)" "" ifneq "$(shell command -v python2.7-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python2.7-config --includes) - PYTHON_LIB ?= $(shell python2.7-config --ldflags) - PYTHON_VERSION ?= $(strip $(shell python2.7 --version 2>&1)) + PYTHON_INCLUDE := $(shell python2.7-config --includes) + PYTHON_LIB := $(shell python2.7-config --ldflags) + PYTHON_VERSION := $(strip $(shell python2.7 --version 2>&1)) endif endif endif @@ -250,17 +269,17 @@ endif ifneq "$(findstring FreeBSD, $(SYS))" "" override CFLAGS += -pthread - override LDFLAGS += -lpthread + override LDFLAGS += -lpthread -lm endif ifneq "$(findstring NetBSD, $(SYS))" "" override CFLAGS += -pthread - override LDFLAGS += -lpthread + override LDFLAGS += -lpthread -lm endif ifneq "$(findstring OpenBSD, $(SYS))" "" override CFLAGS += -pthread - override LDFLAGS += -lpthread + override LDFLAGS += -lpthread -lm endif COMM_HDR = include/alloc-inl.h include/config.h include/debug.h include/types.h @@ -286,8 +305,6 @@ ifeq "$(shell command -v svn >/dev/null && svn proplist . 2>/dev/null && echo 1 IN_REPO=1 endif -CCVER=$(shell cc -v 2>&1|tail -n 1) - ifeq "$(shell echo 'int main() { return 0;}' | $(CC) $(CFLAGS) -fsanitize=address -x c - -o .test2 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1" ASAN_CFLAGS=-fsanitize=address -fstack-protector-all -fno-omit-frame-pointer -DASAN_BUILD ASAN_LDFLAGS=-fsanitize=address -fstack-protector-all -fno-omit-frame-pointer @@ -315,7 +332,7 @@ all: test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_bu @echo Build Summary: @test -e afl-fuzz && echo "[+] afl-fuzz and supporting tools successfully built" || echo "[-] afl-fuzz could not be built, please set CC to a working compiler" @test -e afl-llvm-pass.so && echo "[+] LLVM basic mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" - @test -e SanitizerCoveragePCGUARD.so && echo "[+] LLVM mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" + @test -e SanitizerCoveragePCGUARD.so && echo "[+] LLVM mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-13 and clang-13 or newer, see docs/INSTALL.md" @test -e SanitizerCoverageLTO.so && echo "[+] LLVM LTO mode successfully built" || echo "[-] LLVM LTO mode could not be built, it is optional, if you want it, please install LLVM and LLD 11+. More information at instrumentation/README.lto.md on how to build it" ifneq "$(SYS)" "Darwin" @test -e afl-gcc-pass.so && echo "[+] gcc_mode successfully built" || echo "[-] gcc_mode could not be built, it is optional, install gcc-VERSION-plugin-dev to enable this" @@ -378,6 +395,7 @@ help: @echo @echo Known build environment options: @echo "==========================================" + @echo "PERFORMANCE - compile with performance options that make the binary not transferable to other systems. Recommended!" @echo STATIC - compile AFL++ static @echo "CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md)" @echo ASAN_BUILD - compiles AFL++ with memory sanitizer for debug purposes @@ -388,6 +406,7 @@ help: @echo INTROSPECTION - compile afl-fuzz with mutation introspection @echo NO_PYTHON - disable python support @echo NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for normal fuzzing + @echo "NO_UTF - do not use UTF-8 for line rendering in status screen (fallback to G1 box drawing, of vanilla AFL)" @echo NO_NYX - disable building nyx mode dependencies @echo "NO_CORESIGHT - disable building coresight (arm64 only)" @echo NO_UNICORN_ARM64 - disable building unicorn on arm64 @@ -434,38 +453,38 @@ endif .PHONY: ready ready: - @echo "[+] Everything seems to be working, ready to compile. ($(CCVER))" + @echo "[+] Everything seems to be working, ready to compile. ($(shell $(CC) --version 2>&1|head -n 1))" afl-as: src/afl-as.c include/afl-as.h $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) src/$@.c -o $@ $(LDFLAGS) @ln -sf afl-as as src/afl-performance.o : $(COMM_HDR) src/afl-performance.c include/hash.h - $(CC) $(CFLAGS) $(CFLAGS_OPT) -Iinclude -c src/afl-performance.c -o src/afl-performance.o + $(CC) $(CFLAGS) $(CFLAGS_OPT) $(SPECIAL_PERFORMANCE) -Iinclude -c src/afl-performance.c -o src/afl-performance.o src/afl-common.o : $(COMM_HDR) src/afl-common.c include/common.h - $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-common.c -o src/afl-common.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) -c src/afl-common.c -o src/afl-common.o src/afl-forkserver.o : $(COMM_HDR) src/afl-forkserver.c include/forkserver.h - $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-forkserver.c -o src/afl-forkserver.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) -c src/afl-forkserver.c -o src/afl-forkserver.o src/afl-sharedmem.o : $(COMM_HDR) src/afl-sharedmem.c include/sharedmem.h - $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-sharedmem.c -o src/afl-sharedmem.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) -c src/afl-sharedmem.c -o src/afl-sharedmem.o -afl-fuzz: $(COMM_HDR) include/afl-fuzz.h $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(PYFLAGS) $(LDFLAGS) -lm +afl-fuzz: $(COMM_HDR) include/afl-fuzz.h $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o src/hashmap.c | test_x86 + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) -Wno-shift-count-overflow $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o src/hashmap.c -o $@ $(PYFLAGS) $(LDFLAGS) -lm afl-showmap: src/afl-showmap.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-fuzz-mutators.c src/afl-fuzz-python.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(PYFLAGS) $(LDFLAGS) + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) src/$@.c src/afl-fuzz-mutators.c src/afl-fuzz-python.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(PYFLAGS) $(LDFLAGS) afl-tmin: src/afl-tmin.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(LDFLAGS) afl-analyze: src/afl-analyze.c src/afl-common.o src/afl-sharedmem.o src/afl-performance.o src/afl-forkserver.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-performance.o src/afl-forkserver.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-performance.o src/afl-forkserver.o -o $@ $(LDFLAGS) afl-gotcpu: src/afl-gotcpu.c src/afl-common.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-common.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) src/$@.c src/afl-common.o -o $@ $(LDFLAGS) .PHONY: document document: afl-fuzz-document @@ -482,17 +501,17 @@ unit_maybe_alloc: test/unittests/unit_maybe_alloc.o ./test/unittests/unit_maybe_alloc test/unittests/unit_hash.o : $(COMM_HDR) include/alloc-inl.h test/unittests/unit_hash.c $(AFL_FUZZ_FILES) src/afl-performance.o - @$(CC) $(CFLAGS) $(ASAN_CFLAGS) -c test/unittests/unit_hash.c -o test/unittests/unit_hash.o + @$(CC) $(CFLAGS) $(ASAN_CFLAGS) $(SPECIAL_PERFORMANCE) -c test/unittests/unit_hash.c -o test/unittests/unit_hash.o unit_hash: test/unittests/unit_hash.o src/afl-performance.o - @$(CC) $(CFLAGS) -Wl,--wrap=exit -Wl,--wrap=printf $^ -o test/unittests/unit_hash $(LDFLAGS) $(ASAN_LDFLAGS) -lcmocka + @$(CC) $(CFLAGS) $(SPECIAL_PERFORMANCE) -Wl,--wrap=exit -Wl,--wrap=printf $^ -o test/unittests/unit_hash $(LDFLAGS) $(ASAN_LDFLAGS) -lcmocka ./test/unittests/unit_hash test/unittests/unit_rand.o : $(COMM_HDR) include/alloc-inl.h test/unittests/unit_rand.c $(AFL_FUZZ_FILES) src/afl-performance.o - @$(CC) $(CFLAGS) $(ASAN_CFLAGS) -c test/unittests/unit_rand.c -o test/unittests/unit_rand.o + @$(CC) $(CFLAGS) $(ASAN_CFLAGS) $(SPECIAL_PERFORMANCE) -c test/unittests/unit_rand.c -o test/unittests/unit_rand.o unit_rand: test/unittests/unit_rand.o src/afl-common.o src/afl-performance.o - @$(CC) $(CFLAGS) $(ASAN_CFLAGS) -Wl,--wrap=exit -Wl,--wrap=printf $^ -o test/unittests/unit_rand $(LDFLAGS) $(ASAN_LDFLAGS) -lcmocka + @$(CC) $(CFLAGS) $(ASAN_CFLAGS) $(SPECIAL_PERFORMANCE) -Wl,--wrap=exit -Wl,--wrap=printf $^ -o test/unittests/unit_rand $(LDFLAGS) $(ASAN_LDFLAGS) -lcmocka ./test/unittests/unit_rand test/unittests/unit_list.o : $(COMM_HDR) include/list.h test/unittests/unit_list.c $(AFL_FUZZ_FILES) @@ -641,16 +660,16 @@ endif # -$(MAKE) -C utils/plot_ui -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" - ifeq "$(ARCH)" "aarch64" - ifndef NO_CORESIGHT +ifeq "$(ARCH)" "aarch64" + ifndef NO_CORESIGHT -$(MAKE) -C coresight_mode - endif endif - ifeq "$(SYS)" "Linux" - ifndef NO_NYX +endif +ifeq "$(SYS)" "Linux" +ifndef NO_NYX -cd nyx_mode && ./build_nyx_support.sh - endif - endif +endif +endif -cd qemu_mode && sh ./build_qemu_support.sh ifeq "$(ARCH)" "aarch64" ifndef NO_UNICORN_ARM64 @@ -737,10 +756,10 @@ endif @echo Build Summary: @test -e afl-fuzz && echo "[+] afl-fuzz and supporting tools successfully built" || echo "[-] afl-fuzz could not be built, please set CC to a working compiler" @test -e afl-llvm-pass.so && echo "[+] LLVM basic mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" - @test -e SanitizerCoveragePCGUARD.so && echo "[+] LLVM mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" + @test -e SanitizerCoveragePCGUARD.so && echo "[+] LLVM mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-13 and clang-13 or newer, see docs/INSTALL.md" @test -e SanitizerCoverageLTO.so && echo "[+] LLVM LTO mode successfully built" || echo "[-] LLVM LTO mode could not be built, it is optional, if you want it, please install LLVM 11-14. More information at instrumentation/README.lto.md on how to build it" ifneq "$(SYS)" "Darwin" - test -e afl-gcc-pass.so && echo "[+] gcc_mode successfully built" || echo "[-] gcc_mode could not be built, it is optional, install gcc-VERSION-plugin-dev to enable this" + @test -e afl-gcc-pass.so && echo "[+] gcc_mode successfully built" || echo "[-] gcc_mode could not be built, it is optional, install gcc-VERSION-plugin-dev to enable this" endif ifeq "$(SYS)" "Linux" ifndef NO_NYX @@ -774,7 +793,7 @@ install: all $(MANPAGES) @rm -f $${DESTDIR}$(BIN_PATH)/afl-plot.sh @rm -f $${DESTDIR}$(BIN_PATH)/afl-as @rm -f $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-32.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-64.o $${DESTDIR}$(HELPER_PATH)/afl-gcc-rt.o - @for i in afl-llvm-dict2file.so afl-llvm-lto-instrumentlist.so afl-llvm-pass.so cmplog-instructions-pass.so cmplog-routines-pass.so cmplog-switches-pass.so compare-transform-pass.so libcompcov.so libdislocator.so libnyx.so libqasan.so libtokencap.so SanitizerCoverageLTO.so SanitizerCoveragePCGUARD.so split-compares-pass.so split-switches-pass.so; do echo rm -fv $${DESTDIR}$(HELPER_PATH)/$${i}; done + @for i in afl-llvm-dict2file.so afl-llvm-lto-instrumentlist.so afl-llvm-pass.so cmplog-instructions-pass.so cmplog-routines-pass.so cmplog-switches-pass.so compare-transform-pass.so libcompcov.so libdislocator.so libnyx.so libqasan.so libtokencap.so SanitizerCoverageLTO.so SanitizerCoveragePCGUARD.so split-compares-pass.so split-switches-pass.so injection-pass.so; do echo rm -fv $${DESTDIR}$(HELPER_PATH)/$${i}; done install -m 755 $(PROGS) $(SH_PROGS) $${DESTDIR}$(BIN_PATH) @if [ -f afl-qemu-trace ]; then install -m 755 afl-qemu-trace $${DESTDIR}$(BIN_PATH); fi @if [ -f utils/plot_ui/afl-plot-ui ]; then install -m 755 utils/plot_ui/afl-plot-ui $${DESTDIR}$(BIN_PATH); fi @@ -805,11 +824,12 @@ endif install -m 644 docs/*.md $${DESTDIR}$(DOC_PATH) cp -r testcases/ $${DESTDIR}$(MISC_PATH) cp -r dictionaries/ $${DESTDIR}$(MISC_PATH) + cp injections.dic $${DESTDIR}$(MISC_PATH) .PHONY: uninstall uninstall: - -cd $${DESTDIR}$(BIN_PATH) && rm -f $(PROGS) $(SH_PROGS) afl-cs-proxy afl-qemu-trace afl-plot-ui afl-fuzz-document afl-network-server afl-g* afl-plot.sh afl-as afl-ld-lto afl-c* afl-lto* - -cd $${DESTDIR}$(HELPER_PATH) && rm -f afl-g*.*o afl-llvm-*.*o afl-compiler-*.*o libdislocator.so libtokencap.so libcompcov.so libqasan.so afl-frida-trace.so libnyx.so socketfuzz*.so argvfuzz*.so libAFLDriver.a libAFLQemuDriver.a as afl-as SanitizerCoverage*.so compare-transform-pass.so cmplog-*-pass.so split-*-pass.so dynamic_list.txt + -cd $${DESTDIR}$(BIN_PATH) && rm -f $(PROGS) $(SH_PROGS) afl-cs-proxy afl-qemu-trace afl-plot-ui afl-fuzz-document afl-network-client afl-network-server afl-g* afl-plot.sh afl-as afl-ld-lto afl-c* afl-lto* + -cd $${DESTDIR}$(HELPER_PATH) && rm -f afl-g*.*o afl-llvm-*.*o afl-compiler-*.*o libdislocator.so libtokencap.so libcompcov.so libqasan.so afl-frida-trace.so libnyx.so socketfuzz*.so argvfuzz*.so libAFLDriver.a libAFLQemuDriver.a as afl-as SanitizerCoverage*.so compare-transform-pass.so cmplog-*-pass.so split-*-pass.so dynamic_list.txt injections.dic -rm -rf $${DESTDIR}$(MISC_PATH)/testcases $${DESTDIR}$(MISC_PATH)/dictionaries -sh -c "ls docs/*.md | sed 's|^docs/|$${DESTDIR}$(DOC_PATH)/|' | xargs rm -f" -cd $${DESTDIR}$(MAN_PATH) && rm -f $(MANPAGES) diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index 41face4c..8f06792d 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -11,7 +11,7 @@ # from Laszlo Szekeres. # # Copyright 2015 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,15 +28,17 @@ MAN_PATH ?= $(PREFIX)/share/man/man8 VERSION = $(shell grep '^$(HASH)define VERSION ' ./config.h | cut -d '"' -f2) -CFLAGS ?= -O3 -g -funroll-loops -D_FORTIFY_SOURCE=1 +CFLAGS ?= -O3 -g -funroll-loops +# -D_FORTIFY_SOURCE=1 CFLAGS_SAFE := -Wall -Iinclude -Wno-pointer-sign \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DGCC_VERSION=\"$(GCCVER)\" -DGCC_BINDIR=\"$(GCCBINDIR)\" \ -Wno-unused-function override CFLAGS += $(CFLAGS_SAFE) -CXXFLAGS ?= -O3 -g -funroll-loops -D_FORTIFY_SOURCE=1 -CXXEFLAGS := $(CXXFLAGS) -Wall -std=c++11 +CXXFLAGS ?= -O3 -g -funroll-loops +# -D_FORTIFY_SOURCE=1 +CXXEFLAGS := $(CXXFLAGS) $(CPPFLAGS) -Wall -std=c++11 CC ?= gcc CXX ?= g++ @@ -59,7 +61,7 @@ ifeq "$(findstring Foundation,$(shell $(CC) --version))" "" endif PLUGIN_BASE = "$(shell $(CC) -print-file-name=plugin)" -PLUGIN_FLAGS = -fPIC -fno-rtti -I$(PLUGIN_BASE)/include -I$(PLUGIN_BASE) +PLUGIN_FLAGS = -fPIC -fno-rtti -fno-exceptions -I$(PLUGIN_BASE)/include -I$(PLUGIN_BASE) HASH=\# GCCVER = $(shell $(CC) --version 2>/dev/null | awk 'NR == 1 {print $$NF}') diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index 2bb4e7f8..70c54f1c 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -44,13 +44,14 @@ endif LLVMVER = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/git//' | sed 's/svn//' ) LLVM_MAJOR = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/\..*//' ) -LLVM_MINOR = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/.*\.//' | sed 's/git//' | sed 's/svn//' | sed 's/ .*//' ) -LLVM_UNSUPPORTED = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^[0-2]\.|^3.[0-7]\.' && echo 1 || echo 0 ) -LLVM_TOO_NEW = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[5-9]' && echo 1 || echo 0 ) +LLVM_MINOR = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/.*\.//' | sed 's/git//' | sed 's/svn//' | sed 's/ .*//' | sed 's/rc.*//' ) +LLVM_UNSUPPORTED = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^[0-2]\.|^3.[0-8]\.' && echo 1 || echo 0 ) +LLVM_TOO_NEW = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^19|^2[0-9]' && echo 1 || echo 0 ) +LLVM_TOO_OLD = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^[1-9]\.|^1[012]\.' && echo 1 || echo 0 ) LLVM_NEW_API = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[0-9]' && echo 1 || echo 0 ) LLVM_NEWER_API = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[6-9]' && echo 1 || echo 0 ) -LLVM_10_OK = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[1-9]|^10\.[1-9]|^10\.0.[1-9]' && echo 1 || echo 0 ) -LLVM_HAVE_LTO = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[1-9]' && echo 1 || echo 0 ) +LLVM_13_OK = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[3-9]' && echo 1 || echo 0 ) +LLVM_HAVE_LTO = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[2-9]' && echo 1 || echo 0 ) LLVM_BINDIR = $(shell $(LLVM_CONFIG) --bindir 2>/dev/null) LLVM_LIBDIR = $(shell $(LLVM_CONFIG) --libdir 2>/dev/null) LLVM_STDCXX = gnu++11 @@ -69,6 +70,12 @@ ifeq "$(LLVM_TOO_NEW)" "1" $(warning you are using an in-development llvm version - this might break llvm_mode!) endif +ifeq "$(LLVM_TOO_OLD)" "1" + $(warning you are using an outdated LLVM version! Please use at least LLVM 13 or newer!) + $(shell sleep 2) +endif + +# No switching the meaning of LLVM_TOO_OLD LLVM_TOO_OLD=1 ifeq "$(LLVM_MAJOR)" "9" @@ -87,18 +94,13 @@ ifeq "$(LLVM_NEWER_API)" "1" LLVM_STDCXX = c++17 endif -ifeq "$(LLVM_TOO_OLD)" "1" - $(info [!] llvm_mode detected an old version of llvm, upgrade to at least 9 or preferable 11!) - $(shell sleep 1) -endif - ifeq "$(LLVM_HAVE_LTO)" "1" - $(info [+] llvm_mode detected llvm 11+, enabling afl-lto LTO implementation) + $(info [+] llvm_mode detected llvm 12+, enabling afl-lto LTO implementation) LLVM_LTO = 1 endif ifeq "$(LLVM_LTO)" "0" - $(info [+] llvm_mode detected llvm < 11, afl-lto LTO will not be build.) + $(info [+] llvm_mode detected llvm < 12, afl-lto LTO will not be build.) endif ifeq "$(LLVM_APPLE_XCODE)" "1" @@ -260,7 +262,8 @@ else AFL_CLANG_DEBUG_PREFIX = endif -CFLAGS ?= -O3 -funroll-loops -fPIC -D_FORTIFY_SOURCE=1 +CFLAGS ?= -O3 -funroll-loops -fPIC +# -D_FORTIFY_SOURCE=1 CFLAGS_SAFE := -Wall -g -Wno-cast-qual -Wno-variadic-macros -Wno-pointer-sign \ -I ./include/ -I ./instrumentation/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ @@ -285,7 +288,8 @@ ifdef AFL_TRACE_PC $(info Compile option AFL_TRACE_PC is deprecated, just set AFL_LLVM_INSTRUMENT=PCGUARD to activate when compiling targets ) endif -CXXFLAGS ?= -O3 -funroll-loops -fPIC -D_FORTIFY_SOURCE=1 +CXXFLAGS ?= -O3 -funroll-loops -fPIC +# -D_FORTIFY_SOURCE=1 override CXXFLAGS += -Wall -g -I ./include/ \ -DVERSION=\"$(VERSION)\" -Wno-variadic-macros -Wno-deprecated-copy-with-dtor \ -DLLVM_MINOR=$(LLVM_MINOR) -DLLVM_MAJOR=$(LLVM_MAJOR) @@ -296,7 +300,7 @@ endif ifneq "$(LLVM_CONFIG)" "" CLANG_CFL += -I$(shell dirname $(LLVM_CONFIG))/../include endif -CLANG_CPPFL = `$(LLVM_CONFIG) --cxxflags` -fno-rtti -fPIC $(CXXFLAGS) -Wno-deprecated-declarations +CLANG_CPPFL = `$(LLVM_CONFIG) --cxxflags` -fno-rtti -fno-exceptions -fPIC $(CXXFLAGS) $(CPPFLAGS) -Wno-deprecated-declarations CLANG_LFL = `$(LLVM_CONFIG) --ldflags` $(LDFLAGS) # wasm fuzzing: disable thread-local storage and unset LLVM debug flag @@ -337,7 +341,7 @@ ifeq "$(TEST_MMAP)" "1" endif PROGS_ALWAYS = ./afl-cc ./afl-compiler-rt.o ./afl-compiler-rt-32.o ./afl-compiler-rt-64.o -PROGS = $(PROGS_ALWAYS) ./afl-llvm-pass.so ./SanitizerCoveragePCGUARD.so ./split-compares-pass.so ./split-switches-pass.so ./cmplog-routines-pass.so ./cmplog-instructions-pass.so ./cmplog-switches-pass.so ./afl-llvm-dict2file.so ./compare-transform-pass.so ./afl-ld-lto ./afl-llvm-lto-instrumentlist.so ./SanitizerCoverageLTO.so +PROGS = $(PROGS_ALWAYS) ./afl-llvm-pass.so ./SanitizerCoveragePCGUARD.so ./split-compares-pass.so ./split-switches-pass.so ./cmplog-routines-pass.so ./cmplog-instructions-pass.so ./cmplog-switches-pass.so ./afl-llvm-dict2file.so ./compare-transform-pass.so ./afl-ld-lto ./afl-llvm-lto-instrumentlist.so ./SanitizerCoverageLTO.so ./injection-pass.so # If prerequisites are not given, warn, do not build anything, and exit with code 0 ifeq "$(LLVMVER)" "" @@ -422,7 +426,7 @@ endif $(CXX) $(CLANG_CPPFL) -Wdeprecated -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o ./SanitizerCoveragePCGUARD.so: instrumentation/SanitizerCoveragePCGUARD.so.cc instrumentation/afl-llvm-common.o | test_deps -ifeq "$(LLVM_10_OK)" "1" +ifeq "$(LLVM_13_OK)" "1" -$(CXX) $(CLANG_CPPFL) -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) -Wno-deprecated-copy-dtor -Wdeprecated instrumentation/afl-llvm-common.o endif @@ -465,6 +469,9 @@ endif afl-llvm-dict2file.so: instrumentation/afl-llvm-dict2file.so.cc instrumentation/afl-llvm-common.o | test_deps $(CXX) $(CLANG_CPPFL) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o +./injection-pass.so: instrumentation/injection-pass.cc instrumentation/afl-llvm-common.o | test_deps + $(CXX) $(CLANG_CPPFL) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o + .PHONY: document document: $(CLANG_BIN) -D_AFL_DOCUMENT_MUTATIONS $(CFLAGS_SAFE) $(CPPFLAGS) $(CLANG_CFL) -O3 -Wno-unused-result -fPIC -c instrumentation/afl-compiler-rt.o.c -o ./afl-compiler-rt.o diff --git a/README.md b/README.md index 863c2fce..34d73890 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # American Fuzzy Lop plus plus (AFL++) -<img align="right" src="https://raw.githubusercontent.com/AFLplusplus/Website/master/static/aflpp_bg.svg" alt="AFL++ logo" width="250" heigh="250"> +<img align="right" src="https://raw.githubusercontent.com/AFLplusplus/Website/main/static/aflpp_bg.svg" alt="AFL++ logo" width="250" heigh="250"> -Release version: [4.06c](https://github.com/AFLplusplus/AFLplusplus/releases) +Release version: [4.20c](https://github.com/AFLplusplus/AFLplusplus/releases) -GitHub version: 4.07a +GitHub version: 4.21a Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) @@ -12,12 +12,13 @@ Repository: AFL++ is maintained by: * Marc "van Hauser" Heuse <mh@mh-sec.de> -* Andrea Fioraldi <andreafioraldi@gmail.com> * Dominik Maier <mail@dmnk.co> -* Heiko "hexcoder-" EiĆfeldt <heiko.eissfeldt@hexco.de> +* Andrea Fioraldi <andreafioraldi@gmail.com> +* Heiko "hexcoder-" Eissfeldt <heiko.eissfeldt@hexco.de> +* frida_mode is maintained by @Worksbutnottested * Documentation: Jana Aydinbas <jana.aydinbas@gmail.com> -Originally developed by MichaÅ "lcamtuf" Zalewski. +Originally developed by Michal "lcamtuf" Zalewski. AFL++ is a superior fork to Google's AFL - more speed, more and better mutations, more and better instrumentation, custom module support, etc. @@ -228,7 +229,7 @@ Thank you! (For people sending pull requests - please add yourself to this list Thomas Rooijakkers David Carlier Ruben ten Hove Joey Jiao fuzzah @intrigus-lgtm - Yaakov Saxon + Yaakov Saxon Sergej Schumilo ``` </details> diff --git a/TODO.md b/TODO.md index 2efcefea..20f3425f 100644 --- a/TODO.md +++ b/TODO.md @@ -1,11 +1,24 @@ # TODO list for AFL++ +## Must + + - hardened_usercopy=0 page_alloc.shuffle=0 + - add value_profile but only enable after 15 minutes without finds + - cmplog max items env? + - adapt MOpt to new mutation engine + - Update afl->pending_not_fuzzed for MOpt + - cmplog rtn sanity check on fixed length? currently we ignore the length + - afl-showmap -f support + - afl-fuzz multicore wrapper script + - when trimming then perform crash detection + - cyclomatic complexity: 2 + calls + edges - blocks + + ## Should - - splicing selection weighted? + - afl-crash-analysis - support persistent and deferred fork server in afl-showmap? - better autodetection of shifting runtime timeout values - - Update afl->pending_not_fuzzed for MOpt - afl-plot to support multiple plot_data - parallel builds for source-only targets - get rid of check_binary, replace with more forkserver communication @@ -24,8 +37,7 @@ QEMU mode/FRIDA mode: - non colliding instrumentation - rename qemu specific envs to AFL_QEMU (AFL_ENTRYPOINT, AFL_CODE_START/END, AFL_COMPCOV_LEVEL?) - - add AFL_QEMU_EXITPOINT (maybe multiple?), maybe pointless as there is - persistent mode + - add AFL_QEMU_EXITPOINT (maybe multiple?) ## Ideas diff --git a/afl-addseeds b/afl-addseeds new file mode 100755 index 00000000..bb2843a8 --- /dev/null +++ b/afl-addseeds @@ -0,0 +1,54 @@ +#!/bin/sh + +test -z "$1" -o "$1" = "-h" -o "$1" = "--help" && { + echo Syntax: afl-addseeds -o afl-out-dir [-i seed_file_or_dir] seed_file_or_seed_dir seed_file_or_seed_dir ... + echo + echo Options: + echo " -o afl-out-dir the output directory being used in the fuzzing campaign" + echo " -i seed_file_or_dir file or directory of files to add" + echo + echo Adds new seeds to an existing AFL++ fuzzing campaign. + exit 0 +} + +for TOOL in find ls; do + X=`which $TOOL` + test -n "$X" || { echo "Error: required tool '$TOOL' not found."; exit 1; } +done + +TEST=`printf %06d 123 2>/dev/null` +test "$TEST" = "000123" || { echo "Error: required tool 'printf' not found."; exit 1; } + +OUT= +NEXT= +for i in $*; do + test -n "$NEXT" && { OUT=$i ; NEXT=""; } + test "$i" = "-o" && { NEXT=1; } +done + +test -d "$OUT" || { echo Error: $OUT is not an existing directory; exit 1; } +OK=`ls $OUT/*/fuzzer_stats 2>/dev/null` +test -n "$OK" || { echo "Error: $OUT is not an 'afl-fuzz -o ... ' output directory" ; exit 1; } + +OUTDIR=$OUT/addseeds/queue +mkdir -p "$OUTDIR" 2>/dev/null +test -d "$OUTDIR" || { echo Error: could not create $OUTDIR ; exit 1 ; } + +echo Adding seeds ... +NEXTID=0 +for i in $*; do + test -z "$i" -o "$i" = "$OUT" -o "$i" = "-i" -o "$i" = "-o" || { + find "$i" -type f | while read FILE; do + N=xxx + while [ -n "$N" ]; do + ID=$NEXTID + N=`ls "$OUTDIR/id:$(printf %06d $ID),"* 2>/dev/null` + NEXTID=$(($NEXTID + 1)) + done + FN=`echo "$FILE" | sed 's/.*\///'` + cp -v "$FILE" "$OUTDIR/id:$(printf %06d $ID),time:0,execs:0,orig:$FN" + done + } +done + +echo Done. diff --git a/afl-cmin b/afl-cmin index ae723c1b..a88460a8 100755 --- a/afl-cmin +++ b/afl-cmin @@ -1,15 +1,19 @@ #!/usr/bin/env sh +SYS=$(uname -s) +test "$SYS" = "Darwin" && { + echo Error: afl-cmin does not work on Apple currently. please use afl-cmin.bash instead. + exit 1 +} export AFL_QUIET=1 export ASAN_OPTIONS=detect_leaks=0 THISPATH=`dirname ${0}` export PATH="${THISPATH}:$PATH" awk -f - -- ${@+"$@"} <<'EOF' #!/usr/bin/awk -f - # awk script to minimize a test corpus of input files # # based on afl-cmin bash script written by Michal Zalewski -# rewritten by Heiko EiĆfeldt (hexcoder-) +# rewritten by Heiko Eissfeldt (hexcoder-) # tested with: # gnu awk (x86 Linux) # bsd awk (x86 *BSD) @@ -104,7 +108,7 @@ function usage() { "\n" \ "Execution control settings:\n" \ " -T tasks - how many parallel tasks to run (default: 1, all=nproc)\n" \ -" -f file - location read by the fuzzed program (stdin)\n" \ +" -f file - location read by the fuzzed program (default: stdin)\n" \ " -m megs - memory limit for child process ("mem_limit" MB)\n" \ " -t msec - run time limit for child process (default: 5000)\n" \ " -O - use binary-only instrumentation (FRIDA mode)\n" \ @@ -259,22 +263,20 @@ BEGIN { # Do a sanity check to discourage the use of /tmp, since we can't really # handle this safely from an awk script. - #if (!ENVIRON["AFL_ALLOW_TMP"]) { - # dirlist[0] = in_dir - # dirlist[1] = target_bin - # dirlist[2] = out_dir - # dirlist[3] = stdin_file - # "pwd" | getline dirlist[4] # current directory - # for (dirind in dirlist) { - # dir = dirlist[dirind] - # - # if (dir ~ /^(\/var)?\/tmp/) { - # print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr" - # exit 1 - # } - # } - # delete dirlist - #} + if (!ENVIRON["AFL_ALLOW_TMP"]) { + dirlist[0] = in_dir + dirlist[1] = target_bin + dirlist[2] = out_dir + dirlist[3] = stdin_file + "pwd" | getline dirlist[4] # current directory + for (dirind in dirlist) { + dir = dirlist[dirind] + if (dir ~ /^(\/var)?\/tmp/) { + print "[-] Warning: do not use this script in /tmp or /var/tmp for security reasons." > "/dev/stderr" + } + } + delete dirlist + } if (threads && stdin_file) { print "[-] Error: -T and -f cannot be used together." > "/dev/stderr" @@ -318,7 +320,9 @@ BEGIN { if (!nyx_mode && target_bin && !exists_and_is_executable(target_bin)) { - "command -v "target_bin" 2>/dev/null" | getline tnew + cmd = "command -v "target_bin" 2>/dev/null" + cmd | getline tnew + close(cmd) if (!tnew || !exists_and_is_executable(tnew)) { print "[-] Error: binary '"target_bin"' not found or not executable." > "/dev/stderr" exit 1 @@ -330,6 +334,7 @@ BEGIN { echo "[!] Trying to obtain the map size of the target ..." get_map_size = "AFL_DUMP_MAP_SIZE=1 " target_bin get_map_size | getline mapsize + close(get_map_size) if (mapsize && mapsize > 65535 && mapsize < 100000000) { AFL_MAP_SIZE = "AFL_MAP_SIZE="mapsize" " print "[+] Setting "AFL_MAP_SIZE @@ -359,14 +364,18 @@ BEGIN { system("rm -rf "trace_dir" 2>/dev/null"); system("rm "out_dir"/id[:_]* 2>/dev/null") - "ls "out_dir"/* 2>/dev/null | wc -l" | getline noofentries + cmd = "ls "out_dir"/* 2>/dev/null | wc -l" + cmd | getline noofentries + close(cmd) if (0 == system( "test -d "out_dir" -a "noofentries" -gt 0" )) { print "[-] Error: directory '"out_dir"' exists and is not empty - delete it first." > "/dev/stderr" exit 1 } if (threads) { - "nproc" | getline nproc + cmd = "nproc" + cmd | getline nproc + close(cmd) if (threads == "all") { threads = nproc } else { @@ -386,12 +395,14 @@ BEGIN { if (stdin_file) { # truncate input file printf "" > stdin_file - close( stdin_file ) + close(stdin_file) } # First we look in PATH if (0 == system("command -v afl-showmap >/dev/null 2>&1")) { - "command -v afl-showmap 2>/dev/null" | getline showmap + cmd = "command -v afl-showmap 2>/dev/null" + cmd | getline showmap + close(cmd) } else { # then we look in the current directory if (0 == system("test -x ./afl-showmap")) { @@ -413,13 +424,15 @@ BEGIN { # yuck, gnu stat is option incompatible to bsd stat # we use a heuristic to differentiate between # GNU stat and other stats - "stat --version 2>/dev/null" | getline statversion - if (statversion ~ /GNU coreutils/) { + cmd = "stat --version 2>/dev/null" + cmd | getline statversion + close(cmd) + if (statversion ~ /GNU coreutils/ || statversion ~ /BusyBox/) { stat_format = "-c '%s %n'" # GNU } else { stat_format = "-f '%z %N'" # *BSD, MacOS } - cmdline = "(cd "in_dir" && find . \\( ! -name \".*\" -a -type d \\) -o -type f -exec stat "stat_format" \\{\\} + | sort -k1n -k2r)" + cmdline = "(cd "in_dir" && find . \\( ! -name \".*\" -a -type d \\) -o -type f -exec stat "stat_format" \\{\\} + | sort -k1n -k2r) | grep -Ev '^0'" #cmdline = "ls "in_dir" | (cd "in_dir" && xargs stat "stat_format" 2>/dev/null) | sort -k1n -k2r" #cmdline = "(cd "in_dir" && stat "stat_format" *) | sort -k1n -k2r" #cmdline = "(cd "in_dir" && ls | xargs stat "stat_format" ) | sort -k1n -k2r" @@ -432,6 +445,7 @@ BEGIN { infilesSmallToBigFullMap[infilesSmallToBigFull[i]] = infilesSmallToBig[i] i++ } + close(cmdline) in_count = i first_file = infilesSmallToBigFull[0] @@ -468,6 +482,7 @@ BEGIN { while ((getline < runtest) > 0) { ++first_count } + close(runtest) if (first_count) { print "[+] OK, "first_count" tuples recorded." @@ -480,6 +495,11 @@ BEGIN { } } + if (in_count < threads) { + threads = in_count + print "[!] WARNING: less inputs than threads, reducing threads to "threads" and likely the overhead of threading makes things slower..." + } + # Let's roll! ############################# @@ -488,7 +508,7 @@ BEGIN { if (threads) { - inputsperfile = in_count / threads + inputsperfile = int(in_count / threads) if (in_count % threads) { inputsperfile++; } @@ -513,7 +533,7 @@ BEGIN { if (threads > 1) { - print "[*] Creating " threads " parallel tasks with about " inputsperfile " each." + print "[*] Creating " threads " parallel tasks with about " inputsperfile " items each." for (i = 1; i <= threads; i++) { if (!stdin_file) { @@ -582,6 +602,15 @@ BEGIN { else { print " Processing file "cur"/"in_count } # create path for the trace file from afl-showmap tracefile_path = trace_dir"/"fn + # ensure the file size is not zero + cmd = "du -b \""tracefile_path"\"" + # "ls -l \""tracefile_path"\"" + cmd | getline output + close(cmd) + split(output, result, "\t") + if (result[1] == 0) { + print "[!] WARNING: file "fn" is crashing the target, ignoring..." + } # gather all keys, and count them while ((getline line < tracefile_path) > 0) { key = line @@ -643,6 +672,7 @@ BEGIN { } } close(sortedKeys) + print "" print "[+] Found "tuple_count" unique tuples across "in_count" files." if (out_count == 1) { diff --git a/afl-cmin.bash b/afl-cmin.bash index d390ff65..99ae80d9 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -7,7 +7,7 @@ # # Copyright 2014, 2015 Google Inc. All rights reserved. # -# Copyright 2019-2023 AFLplusplus +# Copyright 2019-2024 AFLplusplus # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -152,6 +152,7 @@ Minimization settings: -e - solve for edge coverage only, ignore hit counts For additional tips, please consult README.md. +This script cannot read filenames that end with a space ' '. Environment variables used: AFL_KEEP_TRACES: leave the temporary <out_dir>\.traces directory @@ -167,29 +168,28 @@ fi # Do a sanity check to discourage the use of /tmp, since we can't really # handle this safely from a shell script. -#if [ "$AFL_ALLOW_TMP" = "" ]; then -# -# echo "$IN_DIR" | grep -qE '^(/var)?/tmp/' -# T1="$?" -# -# echo "$TARGET_BIN" | grep -qE '^(/var)?/tmp/' -# T2="$?" -# -# echo "$OUT_DIR" | grep -qE '^(/var)?/tmp/' -# T3="$?" -# -# echo "$STDIN_FILE" | grep -qE '^(/var)?/tmp/' -# T4="$?" -# -# echo "$PWD" | grep -qE '^(/var)?/tmp/' -# T5="$?" -# -# if [ "$T1" = "0" -o "$T2" = "0" -o "$T3" = "0" -o "$T4" = "0" -o "$T5" = "0" ]; then -# echo "[-] Error: do not use this script in /tmp or /var/tmp." 1>&2 -# exit 1 -# fi -# -#fi +if [ "$AFL_ALLOW_TMP" = "" ]; then + + echo "$IN_DIR" | grep -qE '^(/var)?/tmp/' + T1="$?" + + echo "$TARGET_BIN" | grep -qE '^(/var)?/tmp/' + T2="$?" + + echo "$OUT_DIR" | grep -qE '^(/var)?/tmp/' + T3="$?" + + echo "$STDIN_FILE" | grep -qE '^(/var)?/tmp/' + T4="$?" + + echo "$PWD" | grep -qE '^(/var)?/tmp/' + T5="$?" + + if [ "$T1" = "0" -o "$T2" = "0" -o "$T3" = "0" -o "$T4" = "0" -o "$T5" = "0" ]; then + echo "[-] Warning: do not use this script in /tmp or /var/tmp for security reasons." 1>&2 + fi + +fi # If @@ is specified, but there's no -f, let's come up with a temporary input # file name. @@ -206,7 +206,7 @@ fi # Check for obvious errors. -if [ ! "$T_ARG" = "" -a ! "$F_ARG" = "" -a ! "$NYX_MODE" == 1 ]; then +if [ ! "$T_ARG" = "" -a -n "$F_ARG" -a ! "$NYX_MODE" == 1 ]; then echo "[-] Error: -T and -f can not be used together." 1>&2 exit 1 fi @@ -323,7 +323,7 @@ if [ ! "$T_ARG" = "" ]; then fi fi else - if [ "$F_ARG" = ""]; then + if [ -z "$F_ARG" ]; then echo "[*] Are you aware of the '-T all' parallelize option that massively improves the speed?" fi fi @@ -339,6 +339,13 @@ fi echo "[*] Are you aware that afl-cmin is faster than this afl-cmin.bash script?" echo "[+] Found $IN_COUNT files for minimizing." +if [ -n "$THREADS" ]; then + if [ "$IN_COUNT" -lt "$THREADS" ]; then + THREADS=$IN_COUNT + echo "[!] WARNING: less inputs than threads, reducing threads to $THREADS and likely the overhead of threading makes things slower..." + fi +fi + FIRST_FILE=`ls "$IN_DIR" | head -1` # Make sure that we're not dealing with a directory. @@ -416,10 +423,14 @@ if [ "$THREADS" = "" ]; then ls "$IN_DIR" | while read -r fn; do - CUR=$((CUR+1)) - printf "\\r Processing file $CUR/$IN_COUNT... " + if [ -s "$IN_DIR/$fn" ]; then - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$fn" + CUR=$((CUR+1)) + printf "\\r Processing file $CUR/$IN_COUNT... " + + "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$fn" + + fi done @@ -427,11 +438,15 @@ if [ "$THREADS" = "" ]; then ls "$IN_DIR" | while read -r fn; do - CUR=$((CUR+1)) - printf "\\r Processing file $CUR/$IN_COUNT... " + if [ -s "$IN_DIR/$fn" ]; then + + CUR=$((CUR+1)) + printf "\\r Processing file $CUR/$IN_COUNT... " + + cp "$IN_DIR/$fn" "$STDIN_FILE" + "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@" </dev/null - cp "$IN_DIR/$fn" "$STDIN_FILE" - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@" </dev/null + fi done @@ -453,19 +468,26 @@ else cat $inputs | while read -r fn; do - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$fn" + if [ -s "$IN_DIR/$fn" ]; then + + "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$fn" + + fi done else - STDIN_FILE="$inputs.$$" - cat $inputs | while read -r fn; do + if [ -s "$IN_DIR/$fn" ]; then + STDIN_FILE="$inputs.$$" + cat $inputs | while read -r fn; do - cp "$IN_DIR/$fn" "$STDIN_FILE" - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@" </dev/null + cp "$IN_DIR/$fn" "$STDIN_FILE" + "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@" </dev/null - done + done + + fi fi @@ -479,7 +501,7 @@ else echo "[+] all $THREADS running tasks completed." rm -f ${TMPFILE}* - echo trace dir files: $(ls $TRACE_DIR/*|wc -l) + #echo trace dir files: $(ls $TRACE_DIR/*|wc -l) fi @@ -523,6 +545,8 @@ ls -rS "$IN_DIR" | while read -r fn; do sed "s#\$# $fn#" "$TRACE_DIR/$fn" >>"$TRACE_DIR/.candidate_list" + test -s "$TRACE_DIR/$fn" || echo Warning: $fn is ignored because of crashing the target + done echo diff --git a/afl-persistent-config b/afl-persistent-config index 6d96c196..dede032f 100755 --- a/afl-persistent-config +++ b/afl-persistent-config @@ -2,7 +2,7 @@ # written by jhertz # -test "$1" = "-h" -o "$1" = "-hh" && { +test "$1" = "-h" -o "$1" = "-hh" -o "$1" = "--help" && { echo 'afl-persistent-config' echo echo $0 @@ -17,6 +17,11 @@ test "$1" = "-h" -o "$1" = "-hh" && { exit 0 } +if [ $# -ne 0 ]; then + echo "ERROR: Unknown option(s): $@" + exit 1 +fi + echo echo "WARNING: This scripts makes permanent configuration changes to the system to" echo " increase the performance for fuzzing. As a result, the system also" @@ -33,6 +38,7 @@ fi echo PLATFORM=`uname -s` +ARCH=`uname -m` # check that we're on Mac if [[ "$PLATFORM" = "Darwin" ]] ; then @@ -82,6 +88,13 @@ if [[ "$PLATFORM" = "Darwin" ]] ; then </plist> EOF + if [[ "$ARCH" = "x86_64" ]]; then + echo "Disabling ASLR system wide" + nvram boot-args="no_aslr=1" + else + echo NOTICE: on ARM64 we do not know currently how to disable system wide ASLR, please report if you know how. + fi + echo echo "Reboot and enjoy your fuzzing" exit 0 @@ -98,9 +111,9 @@ if [[ "$PLATFORM" = "Linux" ]] ; then echo "Checks passed." test -d /etc/sysctl.d || echo Error: /etc/sysctl.d directory not found, cannot install shmem config - test -d /etc/sysctl.d -a '!' -e /etc/sysctl.d/99-fuzzing && { - echo "Installing /etc/sysctl.d/99-fuzzing" - cat << EOF > /etc/sysctl.d/99-fuzzing + test -d /etc/sysctl.d -a '!' -e /etc/sysctl.d/99-fuzzing.conf && { + echo "Installing /etc/sysctl.d/99-fuzzing.conf" + cat << EOF > /etc/sysctl.d/99-fuzzing.conf kernel.core_uses_pid=0 kernel.core_pattern=core kernel.randomize_va_space=0 @@ -111,17 +124,26 @@ kernel.sched_latency_ns=250000000 EOF } - grep -E -q '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub 2>/dev/null || echo Error: /etc/default/grub with GRUB_CMDLINE_LINUX_DEFAULT is not present, cannot set boot options - grep -E -q '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub 2>/dev/null && { - grep -E '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub | grep -E -q 'noibrs pcid nopti' || { + grub_try_disable_mitigation () { + KEY="$1" + if ! grep -E "^$KEY=" /etc/default/grub | grep -E -q 'noibrs pcid nopti'; then echo "Configuring performance boot options" - LINE=`grep -E '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub | sed 's/^GRUB_CMDLINE_LINUX_DEFAULT=//' | tr -d '"'` - OPTIONS="$LINE ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs pcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=on pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off srbds=off noexec=off noexec32=off tsx=on tsx=on tsx_async_abort=off mitigations=off audit=0 hardened_usercopy=off ssbd=force-off" - echo Setting boot options in /etc/default/grub to GRUB_CMDLINE_LINUX_DEFAULT=\"$OPTIONS\" - sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$OPTIONS\"|" /etc/default/grub - } + LINE=`grep -E "^$KEY=" /etc/default/grub | sed "s/^$KEY=//" | tr -d '"'` + OPTIONS="$LINE ibpb=off ibrs=off kpti=off l1tf=off spec_rstack_overflow=off mds=off no_stf_barrier noibpb noibrs pcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=on pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off srbds=off noexec=off noexec32=off tsx=on tsx=on tsx_async_abort=off mitigations=off audit=0 hardened_usercopy=off ssbd=force-off" + echo Setting boot options in /etc/default/grub to $KEY=\"$OPTIONS\" + sed -i "s|^$KEY=.*|$KEY=\"$OPTIONS\"|" /etc/default/grub + fi } + + if grep -E -q '^GRUB_CMDLINE_LINUX=' /etc/default/grub || grep -E -q '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub; then + grub_try_disable_mitigation "GRUB_CMDLINE_LINUX_DEFAULT" + # We also overwrite GRUB_CMDLINE_LINUX because some distributions already overwrite GRUB_CMDLINE_LINUX_DEFAULT + grub_try_disable_mitigation "GRUB_CMDLINE_LINUX" + else + echo "Error: /etc/default/grub with GRUB_CMDLINE_LINUX is not present, cannot set boot options" + fi + echo echo "Reboot and enjoy your fuzzing" exit 0 diff --git a/afl-plot b/afl-plot index 230d3bfe..f1f288a3 100755 --- a/afl-plot +++ b/afl-plot @@ -75,8 +75,17 @@ outputdir=`get_abs_path "$2"` if [ ! -f "$inputdir/plot_data" ]; then - echo "[-] Error: input directory is not valid (missing 'plot_data')." 1>&2 - exit 1 + if [ -f "$inputdir/default/plot_data" ]; then + + echo "[-] Error: input directory is not valid (missing 'plot_data'), likely you mean $inputdir/default?" 1>&2 + exit 1 + + else + + echo "[-] Error: input directory is not valid (missing 'plot_data')." 1>&2 + exit 1 + + fi fi @@ -141,7 +150,7 @@ set output '$outputdir/high_freq.png' $GNUPLOT_SETUP plot '$inputdir/plot_data' using 1:4 with filledcurve x1 title 'corpus count' linecolor rgb '#000000' fillstyle transparent solid 0.2 noborder, \\ - '' using 1:3 with filledcurve x1 title 'current fuzz item' linecolor rgb '#f0f0f0' fillstyle transparent solid 0.5 noborder, \\ + '' using 1:3 with filledcurve x1 title 'current item' linecolor rgb '#f0f0f0' fillstyle transparent solid 0.5 noborder, \\ '' using 1:5 with lines title 'pending items' linecolor rgb '#0090ff' linewidth 3, \\ '' using 1:6 with lines title 'pending favs' linecolor rgb '#c00080' linewidth 3, \\ '' using 1:2 with lines title 'cycles done' linecolor rgb '#c000f0' linewidth 3 diff --git a/afl-system-config b/afl-system-config index b50bb06e..7e2cb688 100755 --- a/afl-system-config +++ b/afl-system-config @@ -1,5 +1,5 @@ #!/bin/sh -test "$1" = "-h" -o "$1" = "-hh" && { +test "$1" = "-h" -o "$1" = "-hh" -o "$1" = "--help" && { echo 'afl-system-config by Marc Heuse <mh@mh-sec.de>' echo echo $0 @@ -13,6 +13,10 @@ test "$1" = "-h" -o "$1" = "-hh" && { echo configuration options. exit 0 } +if [ $# -ne 0 ]; then + echo "ERROR: Unknown option(s): $@" + exit 1 +fi DONE= PLATFORM=`uname -s` @@ -21,6 +25,7 @@ echo "WARNING: this reduces the security of the system!" echo if [ '!' "$EUID" = 0 ] && [ '!' `id -u` = 0 ] ; then echo "Warning: you need to be root to run this!" + sleep 1 # we do not exit as other mechanisms exist that allows to do this than # being root. let the errors speak for themselves. fi @@ -110,12 +115,12 @@ if [ "$PLATFORM" = "Darwin" ] ; then sysctl kern.sysv.shmall=131072000 echo Settings applied. echo - if $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') ; then + if $(launchctl list 2>/dev/null | grep -q '\.ReportCrash\>') ; then echo echo Unloading the default crash reporter SL=/System/Library; PL=com.apple.ReportCrash - launchctl unload -w ${SL}/LaunchAgents/${PL}.plist >/dev/null 2>&1 - sudo launchctl unload -w ${SL}/LaunchDaemons/${PL}.Root.plist >/dev/null 2>&1 + sudo -u "$SUDO_USER" launchctl unload -w ${SL}/LaunchAgents/${PL}.plist + launchctl unload -w ${SL}/LaunchDaemons/${PL}.Root.plist echo fi echo It is recommended to disable System Integrity Protection for increased performance. diff --git a/afl-whatsup b/afl-whatsup index 6f29ab24..19841755 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -6,7 +6,7 @@ # Originally written by Michal Zalewski # # Copyright 2015 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,79 +18,123 @@ # instances of afl-fuzz. # -echo "$0 status check tool for afl-fuzz by Michal Zalewski" -echo test "$1" = "-h" -o "$1" = "-hh" && { + echo "$0 status check tool for afl-fuzz by Michal Zalewski" + echo echo "Usage: $0 [-s] [-d] afl_output_directory" echo echo Options: - echo " -s - skip details and output summary results only" echo " -d - include dead fuzzer stats" + echo " -m - just show minimal stats" + echo " -n - no color output" + echo " -s - skip details and output summary results only" echo exit 1 } -unset SUMMARY_ONLY +unset MINIMAL_ONLY +unset NO_COLOR unset PROCESS_DEAD +unset SUMMARY_ONLY +unset RED +unset GREEN +unset YELLOW +unset BLUE +unset NC +unset RESET -while [ "$1" = "-s" -o "$1" = "-d" ]; do - - if [ "$1" = "-s" ]; then - SUMMARY_ONLY=1 - fi +if [ -z "$TERM" ]; then export TERM=vt220; fi +while [ "$1" = "-d" -o "$1" = "-m" -o "$1" = "-n" -o "$1" = "-s" ]; do + if [ "$1" = "-d" ]; then PROCESS_DEAD=1 fi + if [ "$1" = "-m" ]; then + MINIMAL_ONLY=1 + fi + + if [ "$1" = "-n" ]; then + NO_COLOR=1 + fi + + if [ "$1" = "-s" ]; then + SUMMARY_ONLY=1 + fi + shift - + done DIR="$1" -if [ "$DIR" = "" ]; then - - echo "Usage: $0 [-s] [-d] afl_output_directory" 1>&2 +if [ "$DIR" = "" -o "$DIR" = "-h" -o "$DIR" = "--help" ]; then + + echo "$0 status check tool for afl-fuzz by Michal Zalewski" 1>&2 + echo 1>&2 + echo "Usage: $0 [-d] [-m] [-n] [-s] afl_output_directory" 1>&2 echo 1>&2 echo Options: 1>&2 - echo " -s - skip details and output summary results only" 1>&2 echo " -d - include dead fuzzer stats" 1>&2 + echo " -m - just show minimal stats" 1>&2 + echo " -n - no color output" 1>&2 + echo " -s - skip details and output summary results only" 1>&2 echo 1>&2 exit 1 + +fi +if [ -z "$MINIMAL_ONLY" ]; then + echo "$0 status check tool for afl-fuzz by Michal Zalewski" + echo fi cd "$DIR" || exit 1 if [ -d queue ]; then - + echo "[-] Error: parameter is an individual output directory, not a sync dir." 1>&2 exit 1 - + fi -RED=`tput setaf 9 1 1 2>/dev/null` -GREEN=`tput setaf 2 1 1 2>/dev/null` -BLUE=`tput setaf 4 1 1 2>/dev/null` -YELLOW=`tput setaf 11 1 1 2>/dev/null` -NC=`tput sgr0` -RESET="$NC" +BC=`which bc 2>/dev/null` +FUSER=`which fuser 2>/dev/null` -CUR_TIME=`date +%s` +if [ -z "$NO_COLOR" ]; then + RED=`tput setaf 9 1 1 2>/dev/null` + GREEN=`tput setaf 2 1 1 2>/dev/null` + BLUE=`tput setaf 4 1 1 2>/dev/null` + YELLOW=`tput setaf 11 1 1 2>/dev/null` + NC=`tput sgr0` + RESET="$NC" +fi + +PLATFORM=`uname -s` +if [ "$PLATFORM" = "Linux" ] ; then + CUR_TIME=`cat /proc/uptime | awk '{printf "%.0f\n", $1}'` +else + # This will lead to inacurate results but will prevent the script from breaking on platforms other than Linux + CUR_TIME=`date +%s` +fi TMP=`mktemp -t .afl-whatsup-XXXXXXXX` || TMP=`mktemp -p /data/local/tmp .afl-whatsup-XXXXXXXX` || TMP=`mktemp -p /data/local/tmp .afl-whatsup-XXXXXXXX` || exit 1 +trap "rm -f $TMP" 1 2 3 13 15 ALIVE_CNT=0 DEAD_CNT=0 +START_CNT=0 TOTAL_TIME=0 TOTAL_EXECS=0 TOTAL_EPS=0 +TOTAL_EPLM=0 TOTAL_CRASHES=0 TOTAL_HANGS=0 TOTAL_PFAV=0 TOTAL_PENDING=0 +TOTAL_COVERAGE= # Time since last find / crash / hang, formatted as string FMT_TIME="0 days 0 hours" @@ -99,11 +143,11 @@ FMT_CRASH="none seen yet" FMT_HANG="none seen yet" if [ "$SUMMARY_ONLY" = "" ]; then - + echo "Individual fuzzers" echo "==================" echo - + fi fmt_duration() @@ -112,22 +156,22 @@ fmt_duration() if [ $1 -le 0 ]; then return 1 fi - + local duration=$((CUR_TIME - $1)) local days=$((duration / 60 / 60 / 24)) local hours=$(((duration / 60 / 60) % 24)) local minutes=$(((duration / 60) % 60)) local seconds=$((duration % 60)) - + if [ $duration -le 0 ]; then DUR_STRING="0 seconds" - elif [ $duration -eq 1 ]; then + elif [ $duration -eq 1 ]; then DUR_STRING="1 second" - elif [ $days -gt 0 ]; then + elif [ $days -gt 0 ]; then DUR_STRING="$days days, $hours hours" - elif [ $hours -gt 0 ]; then + elif [ $hours -gt 0 ]; then DUR_STRING="$hours hours, $minutes minutes" - elif [ $minutes -gt 0 ]; then + elif [ $minutes -gt 0 ]; then DUR_STRING="$minutes minutes, $seconds seconds" else DUR_STRING="$seconds seconds" @@ -138,112 +182,190 @@ FIRST=true TOTAL_WCOP= TOTAL_LAST_FIND=0 -for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do - - sed 's/^command_line.*$/_skip:1/;s/[ ]*:[ ]*/="/;s/$/"/' "$i" >"$TMP" - . "$TMP" - DIR=$(dirname "$i") - DIR=${DIR##*/} - RUN_UNIX=$run_time - RUN_DAYS=$((RUN_UNIX / 60 / 60 / 24)) - RUN_HRS=$(((RUN_UNIX / 60 / 60) % 24)) - - test -n "$cycles_wo_finds" && { - test -z "$FIRST" && TOTAL_WCOP="${TOTAL_WCOP}/" - TOTAL_WCOP="${TOTAL_WCOP}${cycles_wo_finds}" - FIRST= - } - - if [ "$SUMMARY_ONLY" = "" ]; then - - echo ">>> $afl_banner instance: $DIR ($RUN_DAYS days, $RUN_HRS hrs) fuzzer PID: $fuzzer_pid <<<" - echo - - fi - - if ! kill -0 "$fuzzer_pid" 2>/dev/null; then - +for j in `find . -maxdepth 2 -iname fuzzer_setup | sort`; do + + DIR=$(dirname "$j") + i=$DIR/fuzzer_stats + + if [ -f "$i" ]; then + + IS_STARTING= + IS_DEAD= + sed 's/^command_line.*$/_skip:1/;s/[ ]*:[ ]*/="/;s/$/"/' "$i" >"$TMP" + . "$TMP" + DIRECTORY=$DIR + DIR=${DIR##*/} + RUN_UNIX=$run_time + RUN_DAYS=$((RUN_UNIX / 60 / 60 / 24)) + RUN_HRS=$(((RUN_UNIX / 60 / 60) % 24)) + COVERAGE=$(echo $bitmap_cvg|tr -d %) + if [ -n "$TOTAL_COVERAGE" -a -n "$COVERAGE" -a -n "$BC" ]; then + if [ "$(echo "$TOTAL_COVERAGE < $COVERAGE" | bc)" -eq 1 ]; then + TOTAL_COVERAGE=$COVERAGE + fi + fi + if [ -z "$TOTAL_COVERAGE" ]; then TOTAL_COVERAGE=$COVERAGE ; fi + + test -n "$cycles_wo_finds" && { + test -z "$FIRST" && TOTAL_WCOP="${TOTAL_WCOP}/" + TOTAL_WCOP="${TOTAL_WCOP}${cycles_wo_finds}" + FIRST= + } + if [ "$SUMMARY_ONLY" = "" ]; then - - echo " Instance is dead or running remotely, skipping." + + echo ">>> $afl_banner instance: $DIR ($RUN_DAYS days, $RUN_HRS hrs) fuzzer PID: $fuzzer_pid <<<" echo - + fi - - DEAD_CNT=$((DEAD_CNT + 1)) - last_find=0 - - if [ "$PROCESS_DEAD" = "" ]; then - - continue - + + if ! kill -0 "$fuzzer_pid" 2>/dev/null; then + + if [ -e "$i" ] && [ -e "$j" ] && [ -n "$FUSER" ]; then + + if [ "$i" -ot "$j" ]; then + + # fuzzer_setup is newer than fuzzer_stats, maybe the instance is starting? + TMP_PID=`fuser -v "$DIRECTORY" 2>&1 | grep afl-fuzz` + + if [ -n "$TMP_PID" ]; then + + if [ "$SUMMARY_ONLY" = "" ]; then + + echo " Instance is still starting up, skipping." + echo + + fi + + START_CNT=$((START_CNT + 1)) + last_find=0 + IS_STARTING=1 + + if [ "$PROCESS_DEAD" = "" ]; then + + continue + + fi + + fi + + fi + + fi + + if [ -z "$IS_STARTING" ]; then + + if [ "$SUMMARY_ONLY" = "" ]; then + + echo " Instance is dead or running remotely, skipping." + echo + + fi + + DEAD_CNT=$((DEAD_CNT + 1)) + IS_DEAD=1 + last_find=0 + + if [ "$PROCESS_DEAD" = "" ]; then + + continue + + fi + + fi + fi - - fi - - ALIVE_CNT=$((ALIVE_CNT + 1)) - - EXEC_SEC=0 - test -z "$RUN_UNIX" -o "$RUN_UNIX" = 0 || EXEC_SEC=$((execs_done / RUN_UNIX)) - PATH_PERC=$((cur_item * 100 / corpus_count)) - - TOTAL_TIME=$((TOTAL_TIME + RUN_UNIX)) - TOTAL_EPS=$((TOTAL_EPS + EXEC_SEC)) - TOTAL_EXECS=$((TOTAL_EXECS + execs_done)) - TOTAL_CRASHES=$((TOTAL_CRASHES + saved_crashes)) - TOTAL_HANGS=$((TOTAL_HANGS + saved_hangs)) - TOTAL_PENDING=$((TOTAL_PENDING + pending_total)) - TOTAL_PFAV=$((TOTAL_PFAV + pending_favs)) - - if [ "$last_find" -gt "$TOTAL_LAST_FIND" ]; then - TOTAL_LAST_FIND=$last_find - fi - - if [ "$SUMMARY_ONLY" = "" ]; then - - # Warnings in red - TIMEOUT_PERC=$((exec_timeout * 100 / execs_done)) - if [ $TIMEOUT_PERC -ge 10 ]; then - echo " ${RED}timeout_ratio $TIMEOUT_PERC%${NC}" + + ALIVE_CNT=$((ALIVE_CNT + 1)) + + EXEC_SEC=0 + EXEC_MIN=0 + test -z "$RUN_UNIX" -o "$RUN_UNIX" = 0 || EXEC_SEC=$((execs_done / RUN_UNIX)) + PATH_PERC=$((cur_item * 100 / corpus_count)) + + test "$IS_DEAD" = 1 || EXEC_MIN=$(echo $execs_ps_last_min|sed 's/\..*//') + + TOTAL_TIME=$((TOTAL_TIME + RUN_UNIX)) + TOTAL_EPS=$((TOTAL_EPS + EXEC_SEC)) + TOTAL_EPLM=$((TOTAL_EPLM + EXEC_MIN)) + TOTAL_EXECS=$((TOTAL_EXECS + execs_done)) + TOTAL_CRASHES=$((TOTAL_CRASHES + saved_crashes)) + TOTAL_HANGS=$((TOTAL_HANGS + saved_hangs)) + TOTAL_PENDING=$((TOTAL_PENDING + pending_total)) + TOTAL_PFAV=$((TOTAL_PFAV + pending_favs)) + + if [ "$last_find" -gt "$TOTAL_LAST_FIND" ]; then + TOTAL_LAST_FIND=$last_find fi - - if [ $EXEC_SEC -eq 0 ]; then - echo " ${YELLOW}no data yet, 0 execs/sec${NC}" - elif [ $EXEC_SEC -lt 100 ]; then - echo " ${RED}slow execution, $EXEC_SEC execs/sec${NC}" + + if [ "$SUMMARY_ONLY" = "" ]; then + + # Warnings in red + TIMEOUT_PERC=$((exec_timeout * 100 / execs_done)) + if [ $TIMEOUT_PERC -ge 10 ]; then + echo " ${RED}timeout_ratio $TIMEOUT_PERC%${NC}" + fi + + if [ $EXEC_SEC -eq 0 ]; then + echo " ${YELLOW}no data yet, 0 execs/sec${NC}" + elif [ $EXEC_SEC -lt 100 ]; then + echo " ${RED}slow execution, $EXEC_SEC execs/sec${NC}" + fi + + fmt_duration $last_find && FMT_FIND=$DUR_STRING + fmt_duration $last_crash && FMT_CRASH=$DUR_STRING + fmt_duration $last_hang && FMT_HANG=$DUR_STRING + FMT_CWOP="not available" + test -n "$cycles_wo_finds" && { + test "$cycles_wo_finds" = 0 && FMT_CWOP="$cycles_wo_finds" + test "$cycles_wo_finds" -gt 10 && FMT_CWOP="${YELLOW}$cycles_wo_finds${NC}" + test "$cycles_wo_finds" -gt 50 && FMT_CWOP="${RED}$cycles_wo_finds${NC}" + } + + echo " last_find : $FMT_FIND" + echo " last_crash : $FMT_CRASH" + if [ -z "$MINIMAL_ONLY" ]; then + echo " last_hang : $FMT_HANG" + echo " cycles_wo_finds : $FMT_CWOP" + fi + echo " coverage : $COVERAGE%" + + if [ -z "$MINIMAL_ONLY" ]; then + + CPU_USAGE=$(ps aux | grep -w $fuzzer_pid | grep -v grep | awk '{print $3}') + MEM_USAGE=$(ps aux | grep -w $fuzzer_pid | grep -v grep | awk '{print $4}') + + echo " cpu usage $CPU_USAGE%, memory usage $MEM_USAGE%" + + fi + + echo " cycles $((cycles_done + 1)), lifetime speed $EXEC_SEC execs/sec, items $cur_item/$corpus_count (${PATH_PERC}%)" + + if [ "$saved_crashes" = "0" ]; then + echo " pending $pending_favs/$pending_total, coverage $bitmap_cvg, no crashes yet" + else + echo " pending $pending_favs/$pending_total, coverage $bitmap_cvg, crashes saved $saved_crashes (!)" + fi + + echo + fi - fmt_duration $last_find && FMT_FIND=$DUR_STRING - fmt_duration $last_crash && FMT_CRASH=$DUR_STRING - fmt_duration $last_hang && FMT_HANG=$DUR_STRING - FMT_CWOP="not available" - test -n "$cycles_wo_finds" && { - test "$cycles_wo_finds" = 0 && FMT_CWOP="$cycles_wo_finds" - test "$cycles_wo_finds" -gt 10 && FMT_CWOP="${YELLOW}$cycles_wo_finds${NC}" - test "$cycles_wo_finds" -gt 50 && FMT_CWOP="${RED}$cycles_wo_finds${NC}" - } - - echo " last_find : $FMT_FIND" - echo " last_crash : $FMT_CRASH" - echo " last_hang : $FMT_HANG" - echo " cycles_wo_finds : $FMT_CWOP" - - CPU_USAGE=$(ps aux | grep $fuzzer_pid | grep -v grep | awk '{print $3}') - MEM_USAGE=$(ps aux | grep $fuzzer_pid | grep -v grep | awk '{print $4}') + else - echo " cpu usage $CPU_USAGE%, memory usage $MEM_USAGE%" - echo " cycles $((cycles_done + 1)), lifetime speed $EXEC_SEC execs/sec, items $cur_item/$corpus_count (${PATH_PERC}%)" + if [ ! -e "$i" -a -e "$j" ]; then - if [ "$saved_crashes" = "0" ]; then - echo " pending $pending_favs/$pending_total, coverage $bitmap_cvg, no crashes yet" - else - echo " pending $pending_favs/$pending_total, coverage $bitmap_cvg, crashes saved $saved_crashes (!)" + if [ '!' "$PROCESS_DEAD" = "" ]; then + ALIVE_CNT=$((ALIVE_CNT + 1)) + fi + START_CNT=$((START_CNT + 1)) + last_find=0 + IS_STARTING=1 + fi - echo - fi - + done # Formatting for total time, time since last find, crash, and hang @@ -254,7 +376,7 @@ EXECS_MILLION=$((TOTAL_EXECS / 1000 / 1000)) EXECS_THOUSAND=$((TOTAL_EXECS / 1000 % 1000)) if [ $EXECS_MILLION -gt 9 ]; then FMT_EXECS="$EXECS_MILLION millions" -elif [ $EXECS_MILLION -gt 0 ]; then + elif [ $EXECS_MILLION -gt 0 ]; then FMT_EXECS="$EXECS_MILLION millions, $EXECS_THOUSAND thousands" else FMT_EXECS="$EXECS_THOUSAND thousands" @@ -271,41 +393,60 @@ fmt_duration $TOTAL_LAST_FIND && TOTAL_LAST_FIND=$DUR_STRING test "$TOTAL_TIME" = "0" && TOTAL_TIME=1 if [ "$PROCESS_DEAD" = "" ]; then - + TXT="excluded from stats" - + else - + TXT="included in stats" - ALIVE_CNT=$(($ALIVE_CNT - $DEAD_CNT)) - + ALIVE_CNT=$(($ALIVE_CNT - $DEAD_CNT - $START_CNT)) + fi echo "Summary stats" echo "=============" -echo -echo " Fuzzers alive : $ALIVE_CNT" +if [ -z "$SUMMARY_ONLY" -o -z "$MINIMAL_ONLY" ]; then + echo +fi + +echo " Fuzzers alive : $ALIVE_CNT" + +if [ ! "$START_CNT" = "0" ]; then + echo " Starting up : $START_CNT ($TXT)" +fi if [ ! "$DEAD_CNT" = "0" ]; then - echo " Dead or remote : $DEAD_CNT ($TXT)" + echo " Dead or remote : $DEAD_CNT ($TXT)" fi -echo " Total run time : $FMT_TIME" -echo " Total execs : $FMT_EXECS" -echo " Cumulative speed : $TOTAL_EPS execs/sec" +echo " Total run time : $FMT_TIME" +if [ -z "$MINIMAL_ONLY" ]; then + echo " Total execs : $FMT_EXECS" + echo " Cumulative speed : $TOTAL_EPS execs/sec" + if [ "$ALIVE_CNT" -gt "0" ]; then + echo " Total average speed : $((TOTAL_EPS / ALIVE_CNT)) execs/sec" + fi +fi if [ "$ALIVE_CNT" -gt "0" ]; then - echo " Average speed : $((TOTAL_EPS / ALIVE_CNT)) execs/sec" + echo "Current average speed : $TOTAL_EPLM execs/sec" +fi +if [ -z "$MINIMAL_ONLY" ]; then + echo " Pending items : $TOTAL_PFAV faves, $TOTAL_PENDING total" fi -echo " Pending items : $TOTAL_PFAV faves, $TOTAL_PENDING total" -if [ "$ALIVE_CNT" -gt "1" ]; then - echo " Pending per fuzzer : $((TOTAL_PFAV/ALIVE_CNT)) faves, $((TOTAL_PENDING/ALIVE_CNT)) total (on average)" +if [ "$ALIVE_CNT" -gt "1" -o -n "$MINIMAL_ONLY" ]; then + if [ "$ALIVE_CNT" -gt "0" ]; then + echo " Pending per fuzzer : $((TOTAL_PFAV/ALIVE_CNT)) faves, $((TOTAL_PENDING/ALIVE_CNT)) total (on average)" + fi fi -echo " Crashes saved : $TOTAL_CRASHES" -echo " Hangs saved : $TOTAL_HANGS" -echo "Cycles without finds : $TOTAL_WCOP" -echo " Time without finds : $TOTAL_LAST_FIND" +echo " Coverage reached : ${TOTAL_COVERAGE}%" +echo " Crashes saved : $TOTAL_CRASHES" +if [ -z "$MINIMAL_ONLY" ]; then + echo " Hangs saved : $TOTAL_HANGS" + echo " Cycles without finds : $TOTAL_WCOP" +fi +echo " Time without finds : $TOTAL_LAST_FIND" echo exit 0 diff --git a/benchmark/COMPARISON.md b/benchmark/COMPARISON.md new file mode 100644 index 00000000..e16ef213 --- /dev/null +++ b/benchmark/COMPARISON.md @@ -0,0 +1,13 @@ +|CPU | MHz | threads | singlecore | multicore | afl-*-config | +|----------------------------------------------------|-------|---------|------------|-----------|--------------| +|Raspberry Pi 5 | 2400 | 4 | 25786 | 101114 | both | +|AMD EPYC 7282 16-Core Processor | 3194 | 32 | 87199 | 769001 | both | +|AMD Ryzen 5 PRO 4650G with Radeon Graphics | 3700 | 12 | 95356 | 704840 | both | +|Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz | 4995 | 16 | 120064 | 1168943 | both | +|12th Gen Intel(R) Core(TM) i7-1270P | 4761 | 16 | 149778 | 641219 | both | +|AMD Ryzen 9 5950X 16-Core Processor | 4792 | 32 | 161690 | 2339763 | both | +|Apple Mac Studio M2 Ultra 2023, Linux VM guest | 3500 | 16 | 163570 | 1157465 | both | +|AMD Ryzen 9 6900HS with Radeon Graphics | 4676 | 16 | 62860 | 614404 | system | +|AMD Ryzen 9 6900HS with Radeon Graphics | 4745 | 16 | 135501 | 991133 | both | +|AMD Ryzen 9 7950X3D 16-Core Processor | 5400 | 32 | 71566 | 1566279 | system | +|AMD Ryzen 9 7950X3D 16-Core Processor | 5478 | 32 | 161960 | 2173959 | both | diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..12f4763e --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,59 @@ +# American Fuzzy Lop plus plus (AFL++) + +## benchmarking + +This directory contains benchmarking tools that allow you to compare one machine +with another in terms of raw ability to execute a fuzzing target repeatedly. + +To achieve this, we use a sample program ("test-instr.c") where each path is +equally likely, supply it a single seed, and tell AFL to exit after one run of +deterministic mutations against that seed. + +**Note that this is not a real-world scenario!** +Because the target does basically nothing this is rather a stress test on +Kernel I/O / context switching. +For this reason you will not see a difference if you run the multicore test +with 20 or 40 threads - or even see the performance decline the more threads +(`-f` parameter) you use. In a real-world scenario you can expect to gain +exec/s until 40-60 threads (if you have that many available on your CPU). + +Usage example: + +``` +cd aflplusplus/benchmark +python3 benchmark.py + [*] Ready, starting benchmark... + [*] Compiling the test-instr-persist-shmem fuzzing harness for the benchmark to use. + [*] singlecore test-instr-persist-shmem run 1 of 2, execs/s: 124883.62 + [*] singlecore test-instr-persist-shmem run 2 of 2, execs/s: 126704.93 + [*] Average execs/sec for this test across all runs was: 125794.28 + [*] Using 16 fuzzers for multicore fuzzing (use --fuzzers to override). + [*] multicore test-instr-persist-shmem run 1 of 2, execs/s: 1179822.66 + [*] multicore test-instr-persist-shmem run 2 of 2, execs/s: 1175584.09 + [*] Average execs/sec for this test across all runs was: 1177703.38 + [*] Results have been written to the benchmark-results.jsonl file. + [*] Results have been written to the COMPARISON.md file. +``` + +By default, the script will use a number of parallel fuzzers equal to your +available CPUs/threads (change with `--fuzzers`), and will perform each test +three times and average the result (change with `--runs`). + +The script will use multicore fuzzing instead of singlecore by default (change +with `--mode singlecore`) and use a persistent-mode shared memory harness for +optimal speed (change with `--target test-instr`). + +Feel free to submit the resulting line for your CPU added to the COMPARISON.md +and benchmark-results.jsonl files back to AFL++ in a pull request. + +Each run writes results to [benchmark-results.jsonl](benchmark-results.jsonl) +in [JSON Lines](https://jsonlines.org/) format, ready to be pulled in to other +tools such as [jq -cs](https://jqlang.github.io/jq/) or +[pandas](https://pandas.pydata.org/) for analysis. + +## Data analysis + +There is sample data in [benchmark-results.jsonl](benchmark-results.jsonl), and +a Jupyter notebook for exploring the results and suggesting their meaning at +[benchmark.ipynb](benchmark.ipynb). + diff --git a/benchmark/benchmark-results.jsonl b/benchmark/benchmark-results.jsonl new file mode 100644 index 00000000..91acb6fa --- /dev/null +++ b/benchmark/benchmark-results.jsonl @@ -0,0 +1,424 @@ +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4788.77, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"singlecore": {"execs_per_sec": 9845.64, "execs_total": 98545, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4989.281, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"singlecore": {"execs_per_sec": 125682.73, "execs_total": 1257330, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4799.415, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 120293.77, "execs_total": 1203058, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4703.293, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 231429.96, "execs_total": 2314531, "fuzzers_used": 2}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4800.375, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 346759.33, "execs_total": 3468290, "fuzzers_used": 3}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4915.27, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 455340.06, "execs_total": 4554427, "fuzzers_used": 4}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4701.051, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 568405.15, "execs_total": 5685076, "fuzzers_used": 5}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4704.999, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 678030.96, "execs_total": 6781781, "fuzzers_used": 6}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4800.438, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 782585.04, "execs_total": 7827974, "fuzzers_used": 7}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4794.851, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 893618.35, "execs_total": 8938405, "fuzzers_used": 8}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.383, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 956026.15, "execs_total": 9562791, "fuzzers_used": 9}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.352, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 984942.13, "execs_total": 9853724, "fuzzers_used": 10}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4987.681, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1016758.62, "execs_total": 10172892, "fuzzers_used": 11}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.196, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1053087.9, "execs_total": 10536439, "fuzzers_used": 12}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.211, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1085797.87, "execs_total": 10865305, "fuzzers_used": 13}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.577, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1110640.2, "execs_total": 11114033, "fuzzers_used": 14}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4799.955, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1138984.22, "execs_total": 11397389, "fuzzers_used": 15}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.247, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1168943.19, "execs_total": 11699439, "fuzzers_used": 16}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4994.207, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1135093.91, "execs_total": 11360219, "fuzzers_used": 17}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4994.47, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1160430.45, "execs_total": 11614570, "fuzzers_used": 18}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4991.188, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1155769.97, "execs_total": 11569540, "fuzzers_used": 19}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.63, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1150156.26, "execs_total": 11509407, "fuzzers_used": 20}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.227, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1136873.58, "execs_total": 11377110, "fuzzers_used": 21}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.317, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1112404.25, "execs_total": 11134086, "fuzzers_used": 22}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5000.851, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1143131.72, "execs_total": 11440024, "fuzzers_used": 23}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.261, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1143931.38, "execs_total": 11448786, "fuzzers_used": 24}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.259, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1102090.61, "execs_total": 11028561, "fuzzers_used": 25}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.149, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1116518.7, "execs_total": 11172681, "fuzzers_used": 26}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4801.01, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1099224.19, "execs_total": 11000537, "fuzzers_used": 27}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.448, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1114945.37, "execs_total": 11158802, "fuzzers_used": 28}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.663, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1110889.91, "execs_total": 11118113, "fuzzers_used": 29}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.741, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1058548.28, "execs_total": 10595540, "fuzzers_used": 30}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.852, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1119804.85, "execs_total": 11208645, "fuzzers_used": 31}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.417, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1118828.99, "execs_total": 11197813, "fuzzers_used": 32}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5000.682, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1093426.61, "execs_total": 10942324, "fuzzers_used": 33}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.248, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1108123.59, "execs_total": 11090315, "fuzzers_used": 34}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.053, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1041486.52, "execs_total": 10422413, "fuzzers_used": 35}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.299, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1092395.61, "execs_total": 10932107, "fuzzers_used": 36}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.081, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"singlecore": {"execs_per_sec": 8278.64, "execs_total": 82894, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4994.118, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 90641.62, "execs_total": 906960, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.588, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 178184.19, "execs_total": 1782109, "fuzzers_used": 2}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.204, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 262652.86, "execs_total": 2627228, "fuzzers_used": 3}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.829, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 339119.32, "execs_total": 3391956, "fuzzers_used": 4}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.205, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 420239.94, "execs_total": 4202989, "fuzzers_used": 5}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.0, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 498062.02, "execs_total": 4981367, "fuzzers_used": 6}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5000.407, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 578495.44, "execs_total": 5786691, "fuzzers_used": 7}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5002.997, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 661836.22, "execs_total": 6620265, "fuzzers_used": 8}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.952, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 684808.49, "execs_total": 6850000, "fuzzers_used": 9}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.99, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 707094.65, "execs_total": 7074048, "fuzzers_used": 10}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.003, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 732106.17, "execs_total": 7325352, "fuzzers_used": 11}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.488, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 752910.17, "execs_total": 7533775, "fuzzers_used": 12}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5003.679, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 776179.85, "execs_total": 7767507, "fuzzers_used": 13}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.45, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 797520.58, "execs_total": 7981534, "fuzzers_used": 14}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.313, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 822235.41, "execs_total": 8228941, "fuzzers_used": 15}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.723, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 843897.51, "execs_total": 8445693, "fuzzers_used": 16}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.488, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 843177.15, "execs_total": 8438493, "fuzzers_used": 17}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.299, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 844779.09, "execs_total": 8456834, "fuzzers_used": 18}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.662, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 846060.74, "execs_total": 8465728, "fuzzers_used": 19}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.922, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 847556.23, "execs_total": 8482537, "fuzzers_used": 20}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.098, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 844022.97, "execs_total": 8447616, "fuzzers_used": 21}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.352, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 845818.7, "execs_total": 8464237, "fuzzers_used": 22}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.457, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 844118.27, "execs_total": 8448858, "fuzzers_used": 23}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.019, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 837189.02, "execs_total": 8379746, "fuzzers_used": 24}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.513, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 834712.31, "execs_total": 8354719, "fuzzers_used": 25}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.891, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 836344.12, "execs_total": 8370166, "fuzzers_used": 26}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.494, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 827784.91, "execs_total": 8283782, "fuzzers_used": 27}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.761, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 828641.27, "execs_total": 8293602, "fuzzers_used": 28}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.115, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 826123.67, "execs_total": 8268211, "fuzzers_used": 29}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4993.515, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 817765.77, "execs_total": 8184720, "fuzzers_used": 30}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.555, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 816556.66, "execs_total": 8171816, "fuzzers_used": 31}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.999, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 812661.77, "execs_total": 8132767, "fuzzers_used": 32}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.561, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 805352.16, "execs_total": 8060482, "fuzzers_used": 33}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.938, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 815888.26, "execs_total": 8164454, "fuzzers_used": 34}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.951, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 812348.56, "execs_total": 8129441, "fuzzers_used": 35}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4999.444, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 817278.03, "execs_total": 8178918, "fuzzers_used": 36}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.133, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 91247.98, "execs_total": 912571, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.029, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 177503.74, "execs_total": 1775569, "fuzzers_used": 2}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.516, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 263559.94, "execs_total": 2635863, "fuzzers_used": 3}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.946, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 339880.84, "execs_total": 3399660, "fuzzers_used": 4}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.539, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 418569.46, "execs_total": 4186780, "fuzzers_used": 5}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.53, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 496208.2, "execs_total": 4962992, "fuzzers_used": 6}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.015, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 580870.62, "execs_total": 5809953, "fuzzers_used": 7}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.662, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 662910.24, "execs_total": 6631172, "fuzzers_used": 8}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.8, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 683654.43, "execs_total": 6838092, "fuzzers_used": 9}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.849, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 707555.71, "execs_total": 7078261, "fuzzers_used": 10}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5007.628, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 732211.35, "execs_total": 7325661, "fuzzers_used": 11}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4981.601, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 756121.92, "execs_total": 7565074, "fuzzers_used": 12}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.041, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 774101.97, "execs_total": 7745053, "fuzzers_used": 13}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5004.554, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 796439.54, "execs_total": 7972225, "fuzzers_used": 14}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.433, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 822652.36, "execs_total": 8232836, "fuzzers_used": 15}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.063, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 846458.67, "execs_total": 8473949, "fuzzers_used": 16}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.85, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 847285.31, "execs_total": 8479183, "fuzzers_used": 17}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.627, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 847278.34, "execs_total": 8481577, "fuzzers_used": 18}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5002.007, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 849345.2, "execs_total": 8500890, "fuzzers_used": 19}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.497, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 848498.04, "execs_total": 8491840, "fuzzers_used": 20}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.084, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 848737.28, "execs_total": 8494747, "fuzzers_used": 21}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.872, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 847610.49, "execs_total": 8484864, "fuzzers_used": 22}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.036, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 846329.82, "execs_total": 8471670, "fuzzers_used": 23}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.731, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 839140.26, "execs_total": 8397496, "fuzzers_used": 24}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4988.743, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 843648.98, "execs_total": 8444091, "fuzzers_used": 25}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5004.084, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 835215.19, "execs_total": 8359949, "fuzzers_used": 26}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.828, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 833416.5, "execs_total": 8340275, "fuzzers_used": 27}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.795, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 826512.71, "execs_total": 8272574, "fuzzers_used": 28}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.022, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 828656.04, "execs_total": 8292856, "fuzzers_used": 29}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.939, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 823292.55, "execs_total": 8239885, "fuzzers_used": 30}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.233, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 824657.95, "execs_total": 8252812, "fuzzers_used": 31}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.909, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 817807.44, "execs_total": 8183838, "fuzzers_used": 32}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.834, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 815344.89, "execs_total": 8160193, "fuzzers_used": 33}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.968, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 814327.97, "execs_total": 8149984, "fuzzers_used": 34}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.625, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 819612.64, "execs_total": 8202605, "fuzzers_used": 35}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.404, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 813155.19, "execs_total": 8137546, "fuzzers_used": 36}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5001.911, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"singlecore": {"execs_per_sec": 8391.52, "execs_total": 83932, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4980.444, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"singlecore": {"execs_per_sec": 10754.79, "execs_total": 107720, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5000.011, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 126201.28, "execs_total": 1262139, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4993.941, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 245701.79, "execs_total": 2457750, "fuzzers_used": 2}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4983.297, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 361167.18, "execs_total": 3612273, "fuzzers_used": 3}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4994.008, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 475221.97, "execs_total": 4752815, "fuzzers_used": 4}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.977, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 586393.43, "execs_total": 5865460, "fuzzers_used": 5}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.97, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 690946.36, "execs_total": 6910846, "fuzzers_used": 6}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.017, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 801029.31, "execs_total": 8011774, "fuzzers_used": 7}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.617, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 913876.89, "execs_total": 9140715, "fuzzers_used": 8}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.997, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 946293.38, "execs_total": 9464848, "fuzzers_used": 9}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.162, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 980031.45, "execs_total": 9803628, "fuzzers_used": 10}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4994.223, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1015241.63, "execs_total": 10157948, "fuzzers_used": 11}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4994.761, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1042290.69, "execs_total": 10427527, "fuzzers_used": 12}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.045, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1073567.99, "execs_total": 10739590, "fuzzers_used": 13}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.484, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1119312.88, "execs_total": 11199130, "fuzzers_used": 14}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4994.729, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1156363.75, "execs_total": 11573213, "fuzzers_used": 15}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.146, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1183713.3, "execs_total": 11848245, "fuzzers_used": 16}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.048, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1187603.56, "execs_total": 11886825, "fuzzers_used": 17}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4986.845, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1190369.21, "execs_total": 11914954, "fuzzers_used": 18}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4985.364, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1188828.6, "execs_total": 11902947, "fuzzers_used": 19}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.108, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1187617.46, "execs_total": 11887934, "fuzzers_used": 20}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.754, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1188490.16, "execs_total": 11894967, "fuzzers_used": 21}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.129, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1184138.92, "execs_total": 11850653, "fuzzers_used": 22}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.048, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1189374.23, "execs_total": 11903803, "fuzzers_used": 23}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.261, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1178947.43, "execs_total": 11800850, "fuzzers_used": 24}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.422, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1173540.28, "execs_total": 11743120, "fuzzers_used": 25}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.909, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1168471.78, "execs_total": 11696401, "fuzzers_used": 26}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4966.966, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1169320.61, "execs_total": 11703900, "fuzzers_used": 27}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.207, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1165434.17, "execs_total": 11661131, "fuzzers_used": 28}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4999.554, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1161113.26, "execs_total": 11619771, "fuzzers_used": 29}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4994.822, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1155066.44, "execs_total": 11560147, "fuzzers_used": 30}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.061, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1145196.35, "execs_total": 11461349, "fuzzers_used": 31}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.006, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1151794.28, "execs_total": 11526764, "fuzzers_used": 32}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4995.939, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1151652.84, "execs_total": 11526720, "fuzzers_used": 33}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.002, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1153215.56, "execs_total": 11539780, "fuzzers_used": 34}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.456, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1146882.5, "execs_total": 11478112, "fuzzers_used": 35}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": false, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.183, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1155253.95, "execs_total": 11561694, "fuzzers_used": 36}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4848.974, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 10714.79, "execs_total": 107180, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.353, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 20493.07, "execs_total": 205279, "fuzzers_used": 2}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.198, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 29660.06, "execs_total": 297006, "fuzzers_used": 3}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.015, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 37875.57, "execs_total": 379078, "fuzzers_used": 4}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.975, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 46326.75, "execs_total": 463731, "fuzzers_used": 5}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.579, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 54595.48, "execs_total": 546283, "fuzzers_used": 6}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4983.814, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 62720.98, "execs_total": 628151, "fuzzers_used": 7}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.617, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 70777.99, "execs_total": 708505, "fuzzers_used": 8}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.286, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 74236.02, "execs_total": 743157, "fuzzers_used": 9}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4799.516, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 78134.94, "execs_total": 782272, "fuzzers_used": 10}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4911.536, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 81886.33, "execs_total": 819649, "fuzzers_used": 11}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.199, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 85923.44, "execs_total": 860033, "fuzzers_used": 12}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.447, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 89696.95, "execs_total": 897746, "fuzzers_used": 13}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.496, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 93540.52, "execs_total": 936217, "fuzzers_used": 14}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.936, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 97641.51, "execs_total": 977546, "fuzzers_used": 15}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4991.829, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 101692.65, "execs_total": 1017683, "fuzzers_used": 16}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.489, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 101236.75, "execs_total": 1013188, "fuzzers_used": 17}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.352, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 101006.28, "execs_total": 1011004, "fuzzers_used": 18}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4999.894, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 99952.26, "execs_total": 1000431, "fuzzers_used": 19}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4942.12, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 99798.64, "execs_total": 998795, "fuzzers_used": 20}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.686, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 99018.86, "execs_total": 991012, "fuzzers_used": 21}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.308, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 98600.87, "execs_total": 986643, "fuzzers_used": 22}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.683, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 98634.02, "execs_total": 987082, "fuzzers_used": 23}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.457, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 98352.9, "execs_total": 984071, "fuzzers_used": 24}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.733, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 98118.63, "execs_total": 981865, "fuzzers_used": 25}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4994.474, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 97752.45, "execs_total": 978192, "fuzzers_used": 26}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4853.378, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 97864.07, "execs_total": 979334, "fuzzers_used": 27}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.484, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 97821.8, "execs_total": 978814, "fuzzers_used": 28}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4996.738, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 97564.87, "execs_total": 976335, "fuzzers_used": 29}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.341, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 98508.1, "execs_total": 985853, "fuzzers_used": 30}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.773, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 98238.96, "execs_total": 983062, "fuzzers_used": 31}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.037, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 98363.93, "execs_total": 984411, "fuzzers_used": 32}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.448, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 96758.69, "execs_total": 968157, "fuzzers_used": 33}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.238, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 96327.0, "execs_total": 964046, "fuzzers_used": 34}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4997.619, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 95913.98, "execs_total": 959817, "fuzzers_used": 35}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "i9-9900k, 16GB DDR4-3000, Arch Linux", "compiler": "clang version 16.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4998.076, "cpu_model": "Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz", "cpu_threads": 16}, "targets": {"test-instr": {"multicore": {"execs_per_sec": 95871.39, "execs_total": 959318, "fuzzers_used": 36}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr": {"singlecore": {"execs_per_sec": 5741.89, "execs_total": 57505, "fuzzers_used": 1}}, "test-instr-persist-shmem": {"singlecore": {"execs_per_sec": 163570.34, "execs_total": 1635867, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 164224.43, "execs_total": 1642737, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 167222.58, "execs_total": 1672393, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 306547.24, "execs_total": 3065934, "fuzzers_used": 2}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 436010.2, "execs_total": 4360827, "fuzzers_used": 3}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 536415.92, "execs_total": 5365101, "fuzzers_used": 4}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 622104.43, "execs_total": 6222784, "fuzzers_used": 5}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 729436.2, "execs_total": 7295214, "fuzzers_used": 6}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 820258.88, "execs_total": 8203409, "fuzzers_used": 7}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 884746.31, "execs_total": 8848458, "fuzzers_used": 8}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 947308.55, "execs_total": 9474351, "fuzzers_used": 9}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 985953.62, "execs_total": 9860922, "fuzzers_used": 10}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1009716.71, "execs_total": 10098454, "fuzzers_used": 11}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1041437.1, "execs_total": 10415844, "fuzzers_used": 12}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1068180.17, "execs_total": 10683116, "fuzzers_used": 13}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1108873.82, "execs_total": 11089926, "fuzzers_used": 14}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1134135.0, "execs_total": 11354464, "fuzzers_used": 15}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1157465.79, "execs_total": 11582583, "fuzzers_used": 16}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1122785.14, "execs_total": 11235138, "fuzzers_used": 17}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1094132.3, "execs_total": 10950326, "fuzzers_used": 18}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1041102.04, "execs_total": 10420102, "fuzzers_used": 19}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1022474.0, "execs_total": 10236560, "fuzzers_used": 20}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 960681.48, "execs_total": 9618077, "fuzzers_used": 21}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 853680.22, "execs_total": 8545665, "fuzzers_used": 22}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 799719.75, "execs_total": 8005071, "fuzzers_used": 23}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 797512.71, "execs_total": 7983371, "fuzzers_used": 24}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 659476.15, "execs_total": 6601599, "fuzzers_used": 25}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 560625.96, "execs_total": 5612503, "fuzzers_used": 26}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 537839.62, "execs_total": 5381649, "fuzzers_used": 27}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 510072.53, "execs_total": 5106056, "fuzzers_used": 28}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 408667.49, "execs_total": 4091795, "fuzzers_used": 29}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 453849.79, "execs_total": 4542311, "fuzzers_used": 30}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 405935.72, "execs_total": 4064268, "fuzzers_used": 31}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 579312.77, "execs_total": 5798912, "fuzzers_used": 32}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 470961.79, "execs_total": 4715503, "fuzzers_used": 33}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 436380.3, "execs_total": 4368099, "fuzzers_used": 34}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 439819.17, "execs_total": 4405705, "fuzzers_used": 35}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "Apple Mac Studio M2 Ultra 2023, Linux VM guest, 16 threads assigned to VM", "compiler": "Ubuntu clang version 16.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3500.0, "cpu_model": "Apple Mac Studio M2 Ultra 2023", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 407460.31, "execs_total": 4084528, "fuzzers_used": 36}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3514.326, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 119469.35, "execs_total": 1194813, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.748, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 237177.2, "execs_total": 2372250, "fuzzers_used": 2}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3455.647, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 358305.9, "execs_total": 3583655, "fuzzers_used": 3}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.67, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 475974.21, "execs_total": 4760218, "fuzzers_used": 4}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.813, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 594372.12, "execs_total": 5944793, "fuzzers_used": 5}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3584.545, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 711732.18, "execs_total": 7118626, "fuzzers_used": 6}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.377, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 824314.1, "execs_total": 8245020, "fuzzers_used": 7}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.535, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 936358.89, "execs_total": 9365349, "fuzzers_used": 8}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3469.977, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1010050.77, "execs_total": 10102421, "fuzzers_used": 9}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.644, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1087333.72, "execs_total": 10875294, "fuzzers_used": 10}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3473.935, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1180500.37, "execs_total": 11807345, "fuzzers_used": 11}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3334.193, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1284695.8, "execs_total": 12849848, "fuzzers_used": 12}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3436.186, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1377659.89, "execs_total": 13779252, "fuzzers_used": 13}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.27, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1471828.49, "execs_total": 14721973, "fuzzers_used": 14}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3466.893, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1557812.41, "execs_total": 15581135, "fuzzers_used": 15}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3561.127, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1634678.08, "execs_total": 16349952, "fuzzers_used": 16}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.848, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1518908.2, "execs_total": 15192488, "fuzzers_used": 17}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3591.34, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1470513.71, "execs_total": 14709207, "fuzzers_used": 18}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.619, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1414625.05, "execs_total": 14156400, "fuzzers_used": 19}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.99, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1355481.53, "execs_total": 13565462, "fuzzers_used": 20}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.232, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1292684.55, "execs_total": 12934801, "fuzzers_used": 21}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3442.34, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1234478.66, "execs_total": 12352256, "fuzzers_used": 22}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.796, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1174550.37, "execs_total": 11752094, "fuzzers_used": 23}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3494.124, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1125218.66, "execs_total": 11258330, "fuzzers_used": 24}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3350.261, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1022021.81, "execs_total": 10226548, "fuzzers_used": 25}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.929, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 990339.75, "execs_total": 9908883, "fuzzers_used": 26}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3484.153, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 953861.38, "execs_total": 9543479, "fuzzers_used": 27}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3393.24, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 942151.65, "execs_total": 9426176, "fuzzers_used": 28}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3434.881, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 927072.1, "execs_total": 9275954, "fuzzers_used": 29}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3444.453, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 908669.71, "execs_total": 9092225, "fuzzers_used": 30}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3442.593, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 893432.26, "execs_total": 8938840, "fuzzers_used": 31}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3380.389, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 876618.01, "execs_total": 8770325, "fuzzers_used": 32}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3586.135, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 834676.33, "execs_total": 8350992, "fuzzers_used": 33}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.956, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 830200.25, "execs_total": 8306463, "fuzzers_used": 34}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.94, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 821667.96, "execs_total": 8220135, "fuzzers_used": 35}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3591.052, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 829075.87, "execs_total": 8294543, "fuzzers_used": 36}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3573.541, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 814422.62, "execs_total": 8148191, "fuzzers_used": 37}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.902, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 806770.85, "execs_total": 8071030, "fuzzers_used": 38}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3488.496, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 794433.8, "execs_total": 7947600, "fuzzers_used": 39}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3470.314, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 781022.61, "execs_total": 7813248, "fuzzers_used": 40}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.761, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 754394.26, "execs_total": 7546321, "fuzzers_used": 41}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.125, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 763116.33, "execs_total": 7634125, "fuzzers_used": 42}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.437, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 759323.54, "execs_total": 7596118, "fuzzers_used": 43}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.079, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 764198.14, "execs_total": 7644920, "fuzzers_used": 44}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.619, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 757777.51, "execs_total": 7580317, "fuzzers_used": 45}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3425.09, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 749357.06, "execs_total": 7496189, "fuzzers_used": 46}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.567, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 732083.87, "execs_total": 7323543, "fuzzers_used": 47}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.365, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 721133.28, "execs_total": 7214084, "fuzzers_used": 48}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.699, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 658925.82, "execs_total": 6591967, "fuzzers_used": 49}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.889, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 659890.97, "execs_total": 6601888, "fuzzers_used": 50}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3381.676, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 655176.63, "execs_total": 6554987, "fuzzers_used": 51}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3587.51, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 660889.12, "execs_total": 6612265, "fuzzers_used": 52}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3546.407, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 651803.54, "execs_total": 6520961, "fuzzers_used": 53}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3439.83, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 659012.17, "execs_total": 6593396, "fuzzers_used": 54}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3387.899, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 660016.18, "execs_total": 6603558, "fuzzers_used": 55}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3444.077, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 655931.36, "execs_total": 6561865, "fuzzers_used": 56}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.775, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 618906.23, "execs_total": 6192465, "fuzzers_used": 57}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.33, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 614008.28, "execs_total": 6143464, "fuzzers_used": 58}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.487, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 622400.85, "execs_total": 6227304, "fuzzers_used": 59}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.123, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 624883.06, "execs_total": 6251875, "fuzzers_used": 60}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3590.657, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 628668.94, "execs_total": 6289966, "fuzzers_used": 61}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.335, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 628892.17, "execs_total": 6292361, "fuzzers_used": 62}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.368, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 622065.07, "execs_total": 6224119, "fuzzers_used": 63}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3413.262, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 625528.06, "execs_total": 6258762, "fuzzers_used": 64}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.18, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 602248.19, "execs_total": 6025927, "fuzzers_used": 65}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3591.981, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 597615.89, "execs_total": 5979708, "fuzzers_used": 66}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3600.012, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 607270.98, "execs_total": 6076233, "fuzzers_used": 67}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3507.753, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 608945.09, "execs_total": 6092446, "fuzzers_used": 68}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.845, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 611736.03, "execs_total": 6121207, "fuzzers_used": 69}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3412.629, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 615031.23, "execs_total": 6153592, "fuzzers_used": 70}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3443.261, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 608202.64, "execs_total": 6084885, "fuzzers_used": 71}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.439, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 614339.09, "execs_total": 6146152, "fuzzers_used": 72}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3379.556, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 587046.59, "execs_total": 5873881, "fuzzers_used": 73}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.574, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 587238.27, "execs_total": 5875646, "fuzzers_used": 74}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.098, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 594097.56, "execs_total": 5944036, "fuzzers_used": 75}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3587.762, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 598450.35, "execs_total": 5987756, "fuzzers_used": 76}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.629, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 600430.29, "execs_total": 6007598, "fuzzers_used": 77}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3362.161, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 602014.19, "execs_total": 6023649, "fuzzers_used": 78}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3588.173, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 606146.9, "execs_total": 6065033, "fuzzers_used": 79}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3591.159, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 599360.46, "execs_total": 5997023, "fuzzers_used": 80}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3503.299, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 574792.78, "execs_total": 5751470, "fuzzers_used": 81}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3584.593, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 578265.29, "execs_total": 5785927, "fuzzers_used": 82}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3401.073, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 589985.07, "execs_total": 5903506, "fuzzers_used": 83}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3468.764, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 589281.87, "execs_total": 5895767, "fuzzers_used": 84}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3466.115, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 596581.77, "execs_total": 5969747, "fuzzers_used": 85}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3590.706, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 589017.68, "execs_total": 5893108, "fuzzers_used": 86}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3521.556, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 593403.75, "execs_total": 5937422, "fuzzers_used": 87}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.254, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 601611.06, "execs_total": 6019864, "fuzzers_used": 88}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.211, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 576056.15, "execs_total": 5763322, "fuzzers_used": 89}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.489, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 576151.97, "execs_total": 5764687, "fuzzers_used": 90}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.444, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 583769.1, "execs_total": 5841115, "fuzzers_used": 91}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3446.364, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 585285.47, "execs_total": 5856103, "fuzzers_used": 92}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3562.852, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 581524.67, "execs_total": 5818808, "fuzzers_used": 93}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.403, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 596383.31, "execs_total": 5967460, "fuzzers_used": 94}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3421.421, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 596239.29, "execs_total": 5965882, "fuzzers_used": 95}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3276.519, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 595382.67, "execs_total": 5957136, "fuzzers_used": 96}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.029, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 586144.68, "execs_total": 5865411, "fuzzers_used": 97}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3590.48, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 579467.06, "execs_total": 5798123, "fuzzers_used": 98}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.89, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 572801.45, "execs_total": 5731838, "fuzzers_used": 99}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.31, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 573916.1, "execs_total": 5742901, "fuzzers_used": 100}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3589.943, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 565823.06, "execs_total": 5660910, "fuzzers_used": 101}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3391.191, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 561854.84, "execs_total": 5621778, "fuzzers_used": 102}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3372.775, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 562717.02, "execs_total": 5630085, "fuzzers_used": 103}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3365.142, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 559273.67, "execs_total": 5596400, "fuzzers_used": 104}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3591.44, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 553209.58, "execs_total": 5535044, "fuzzers_used": 105}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3563.12, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 547678.42, "execs_total": 5480061, "fuzzers_used": 106}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3477.381, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 552316.36, "execs_total": 5526570, "fuzzers_used": 107}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.467, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 545257.97, "execs_total": 5455157, "fuzzers_used": 108}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3344.258, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 549190.03, "execs_total": 5495511, "fuzzers_used": 109}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3421.467, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 546845.0, "execs_total": 5472086, "fuzzers_used": 110}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.157, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 545239.46, "execs_total": 5455236, "fuzzers_used": 111}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.389, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 543139.24, "execs_total": 5434484, "fuzzers_used": 112}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3461.931, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 543252.43, "execs_total": 5435319, "fuzzers_used": 113}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3354.728, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 538720.77, "execs_total": 5390315, "fuzzers_used": 114}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.185, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 536681.55, "execs_total": 5369963, "fuzzers_used": 115}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.862, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 540956.43, "execs_total": 5412850, "fuzzers_used": 116}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.403, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 536348.84, "execs_total": 5367054, "fuzzers_used": 117}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.449, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 534734.41, "execs_total": 5350358, "fuzzers_used": 118}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.736, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 536060.28, "execs_total": 5363892, "fuzzers_used": 119}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3590.738, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 533480.83, "execs_total": 5338193, "fuzzers_used": 120}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.482, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 514271.98, "execs_total": 5145571, "fuzzers_used": 121}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.864, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 503271.79, "execs_total": 5035794, "fuzzers_used": 122}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3586.097, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 496011.52, "execs_total": 4963063, "fuzzers_used": 123}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3587.507, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 490784.42, "execs_total": 4910734, "fuzzers_used": 124}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.718, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 488441.09, "execs_total": 4887140, "fuzzers_used": 125}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.035, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 481281.33, "execs_total": 4815386, "fuzzers_used": 126}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.332, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 469294.96, "execs_total": 4695183, "fuzzers_used": 127}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3589.346, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 465563.78, "execs_total": 4657841, "fuzzers_used": 128}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.943, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 459922.67, "execs_total": 4601391, "fuzzers_used": 129}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3280.928, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 459384.3, "execs_total": 4596590, "fuzzers_used": 130}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.875, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 453310.58, "execs_total": 4535383, "fuzzers_used": 131}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3600.179, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 460246.7, "execs_total": 4604954, "fuzzers_used": 132}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3601.396, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 457201.82, "execs_total": 4574474, "fuzzers_used": 133}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3600.942, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 452487.43, "execs_total": 4527226, "fuzzers_used": 134}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3458.573, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 450514.18, "execs_total": 4507745, "fuzzers_used": 135}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.922, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 449479.52, "execs_total": 4496843, "fuzzers_used": 136}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.911, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 444691.06, "execs_total": 4449491, "fuzzers_used": 137}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.654, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 443497.81, "execs_total": 4437339, "fuzzers_used": 138}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.626, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 437981.1, "execs_total": 4382263, "fuzzers_used": 139}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.124, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 443055.68, "execs_total": 4432987, "fuzzers_used": 140}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.978, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 438908.41, "execs_total": 4391393, "fuzzers_used": 141}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3453.125, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 442841.02, "execs_total": 4430878, "fuzzers_used": 142}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3214.708, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 441891.92, "execs_total": 4421776, "fuzzers_used": 143}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.764, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 441860.76, "execs_total": 4421068, "fuzzers_used": 144}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3443.44, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 426935.73, "execs_total": 4272029, "fuzzers_used": 145}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3586.383, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 427322.41, "execs_total": 4275938, "fuzzers_used": 146}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3424.014, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 426914.69, "execs_total": 4271924, "fuzzers_used": 147}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.58, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 433246.64, "execs_total": 4335165, "fuzzers_used": 148}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.546, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 435016.77, "execs_total": 4352822, "fuzzers_used": 149}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.587, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 432197.7, "execs_total": 4324740, "fuzzers_used": 150}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3537.464, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 434928.88, "execs_total": 4351767, "fuzzers_used": 151}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.135, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 435174.29, "execs_total": 4354184, "fuzzers_used": 152}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3371.959, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 426852.22, "execs_total": 4271150, "fuzzers_used": 153}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.413, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 431241.89, "execs_total": 4315307, "fuzzers_used": 154}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3590.69, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 430842.14, "execs_total": 4311025, "fuzzers_used": 155}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3591.29, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 434156.3, "execs_total": 4344575, "fuzzers_used": 156}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3583.517, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 430896.1, "execs_total": 4311642, "fuzzers_used": 157}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.926, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 435704.89, "execs_total": 4360326, "fuzzers_used": 158}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.395, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 438155.8, "execs_total": 4384203, "fuzzers_used": 159}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3396.521, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 442883.53, "execs_total": 4432039, "fuzzers_used": 160}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.95, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 433993.37, "execs_total": 4342838, "fuzzers_used": 161}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.614, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 437174.96, "execs_total": 4374708, "fuzzers_used": 162}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.894, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 435745.93, "execs_total": 4360320, "fuzzers_used": 163}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.633, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 441564.58, "execs_total": 4418619, "fuzzers_used": 164}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.069, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 445500.18, "execs_total": 4457810, "fuzzers_used": 165}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3581.223, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 445887.53, "execs_total": 4461995, "fuzzers_used": 166}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.249, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 443509.97, "execs_total": 4438012, "fuzzers_used": 167}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.106, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 446851.67, "execs_total": 4471572, "fuzzers_used": 168}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3417.764, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 447685.22, "execs_total": 4479536, "fuzzers_used": 169}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3589.058, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 446730.72, "execs_total": 4470322, "fuzzers_used": 170}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.116, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 448668.48, "execs_total": 4489967, "fuzzers_used": 171}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3587.905, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 450972.11, "execs_total": 4513110, "fuzzers_used": 172}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.114, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 450615.23, "execs_total": 4509271, "fuzzers_used": 173}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.851, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 458016.89, "execs_total": 4583318, "fuzzers_used": 174}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.106, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 460677.5, "execs_total": 4609716, "fuzzers_used": 175}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3374.143, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 460763.9, "execs_total": 4610640, "fuzzers_used": 176}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.42, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 452298.55, "execs_total": 4526006, "fuzzers_used": 177}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.801, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 456748.89, "execs_total": 4570571, "fuzzers_used": 178}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.709, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 451289.94, "execs_total": 4516046, "fuzzers_used": 179}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.769, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 463235.15, "execs_total": 4635628, "fuzzers_used": 180}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3330.854, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 464366.11, "execs_total": 4646649, "fuzzers_used": 181}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.585, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 469453.17, "execs_total": 4697909, "fuzzers_used": 182}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.242, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 467300.47, "execs_total": 4676077, "fuzzers_used": 183}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.952, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 475115.57, "execs_total": 4754150, "fuzzers_used": 184}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3583.539, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 472179.98, "execs_total": 4724913, "fuzzers_used": 185}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3598.57, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 465528.62, "execs_total": 4658439, "fuzzers_used": 186}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3587.126, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 476194.69, "execs_total": 4765385, "fuzzers_used": 187}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3423.033, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 475886.86, "execs_total": 4762069, "fuzzers_used": 188}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.32, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 473599.91, "execs_total": 4739128, "fuzzers_used": 189}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3597.599, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 476949.52, "execs_total": 4772500, "fuzzers_used": 190}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3437.101, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 474259.76, "execs_total": 4745505, "fuzzers_used": 191}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "AWS EC2 r6a.48xlarge spot instance", "compiler": "clang version 15.0.7 (Amazon Linux 15.0.7-3.amzn2023.0.1)", "target_arch": "x86_64-amazon-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3599.17, "cpu_model": "AMD EPYC 7R13 Processor", "cpu_threads": 192}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 479848.23, "execs_total": 4801111, "fuzzers_used": 192}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Debian clang version 17.0.4 (++20231031083102+309d55140c46-1~exp1~20231031083155.63)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4761.063, "cpu_model": "12th Gen Intel(R) Core(TM) i7-1270P", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 641219.02, "execs_total": 19251242, "fuzzers_used": 16}, "singlecore": {"execs_per_sec": 149778.22, "execs_total": 4493796, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Ubuntu clang version 17.0.2 (++20231003073128+b2417f51dbbd-1~exp1~20231003073233.51)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3193.942, "cpu_model": "AMD EPYC 7282 16-Core Processor", "cpu_threads": 64}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 769000.8, "execs_total": 23084516, "fuzzers_used": 32}, "singlecore": {"execs_per_sec": 87198.85, "execs_total": 2616227, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.08a", "comment": "", "compiler": "Ubuntu clang version 14.0.0-1ubuntu1.1", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 3700.0, "cpu_model": "AMD Ryzen 5 PRO 4650G with Radeon Graphics", "cpu_threads": 12}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 704840.16, "execs_total": 21163992, "fuzzers_used": 12}, "singlecore": {"execs_per_sec": 95356.14, "execs_total": 2862114, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.09a", "comment": "", "compiler": "Debian clang version 14.0.6", "target_arch": "aarch64-unknown-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 2400.0, "cpu_model": "Raspberry Pi 5", "cpu_threads": 4}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 101114.23, "execs_total": 3036637, "fuzzers_used": 4}, "singlecore": {"execs_per_sec": 25786.11, "execs_total": 774460, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.07a", "comment": "", "compiler": "Debian clang version 17.0.0 (++20230417071830+ae77aceba5ad-1~exp1~20230417071935.630)", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4792.073, "cpu_model": "AMD Ryzen 9 5950X 16-Core Processor", "cpu_threads": 32}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 2339762.91, "execs_total": 70253164, "fuzzers_used": 32}, "singlecore": {"execs_per_sec": 161690.07, "execs_total": 4851838, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.10c", "comment": "", "compiler": "clang version 17.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4675.949, "cpu_model": "AMD Ryzen 9 6900HS with Radeon Graphics", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 614403.91, "execs_total": 18435083, "fuzzers_used": 16}, "singlecore": {"execs_per_sec": 62859.9, "execs_total": 1886111, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.10c", "comment": "", "compiler": "clang version 17.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 4744.522, "cpu_model": "AMD Ryzen 9 6900HS with Radeon Graphics", "cpu_threads": 16}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 991132.96, "execs_total": 29737588, "fuzzers_used": 16}, "singlecore": {"execs_per_sec": 135501.07, "execs_total": 4066116, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": false, "afl_system_config": true, "afl_version": "++4.10c", "comment": "", "compiler": "Ubuntu clang version 14.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5399.822, "cpu_model": "AMD Ryzen 9 7950X3D 16-Core Processor", "cpu_threads": 32}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 1566279.42, "execs_total": 46994452, "fuzzers_used": 32}, "singlecore": {"execs_per_sec": 71565.56, "execs_total": 2147396, "fuzzers_used": 1}}}} +{"config": {"afl_persistent_config": true, "afl_system_config": true, "afl_version": "++4.10c", "comment": "", "compiler": "clang version 17.0.6", "target_arch": "x86_64-pc-linux-gnu"}, "hardware": {"cpu_fastest_core_mhz": 5478.258, "cpu_model": "AMD Ryzen 9 7950X3D 16-Core Processor", "cpu_threads": 32}, "targets": {"test-instr-persist-shmem": {"multicore": {"execs_per_sec": 2173959.15, "execs_total": 65229513, "fuzzers_used": 32}, "singlecore": {"execs_per_sec": 161960.29, "execs_total": 4859457, "fuzzers_used": 1}}}} diff --git a/benchmark/benchmark.ipynb b/benchmark/benchmark.ipynb new file mode 100644 index 00000000..aea2e0f1 --- /dev/null +++ b/benchmark/benchmark.ipynb @@ -0,0 +1,1445 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 142, + "metadata": {}, + "outputs": [], + "source": [ + "# benchmark.ipynb\n", + "# Part of the aflplusplus project, requires an ipynb (Jupyter) editor or viewer.\n", + "# Author: Chris Ball <chris@printf.net>\n", + "import json\n", + "import pandas as pd\n", + "with open(\"benchmark-results.jsonl\") as f:\n", + " lines = f.read().splitlines()\n", + "json_lines = [json.loads(line) for line in lines]\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Translate the JSON Lines entries into a single pandas DataFrame\n", + "\n", + "We have JSON Lines in [benchmark-results.jsonl](benchmark-results.jsonl) that look like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 143, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"config\": {\n", + " \"afl_persistent_config\": true,\n", + " \"afl_system_config\": true,\n", + " \"afl_version\": \"++4.09a\",\n", + " \"comment\": \"i9-9900k, 16GB DDR4-3000, Arch Linux\",\n", + " \"compiler\": \"clang version 16.0.6\",\n", + " \"target_arch\": \"x86_64-pc-linux-gnu\"\n", + " },\n", + " \"hardware\": {\n", + " \"cpu_fastest_core_mhz\": 4788.77,\n", + " \"cpu_model\": \"Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz\",\n", + " \"cpu_threads\": 16\n", + " },\n", + " \"targets\": {\n", + " \"test-instr\": {\n", + " \"singlecore\": {\n", + " \"execs_per_sec\": 9845.64,\n", + " \"execs_total\": 98545,\n", + " \"fuzzers_used\": 1\n", + " }\n", + " }\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "print(json.dumps(json.loads(lines[0]), indent=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The [pd.json_normalize()](https://pandas.pydata.org/docs/reference/api/pandas.json_normalize.html]) method translates this into a flat table that we can perform queries against:" + ] + }, + { + "cell_type": "code", + "execution_count": 144, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>config.afl_persistent_config</th>\n", + " <th>config.afl_system_config</th>\n", + " <th>config.afl_version</th>\n", + " <th>config.comment</th>\n", + " <th>config.compiler</th>\n", + " <th>config.target_arch</th>\n", + " <th>hardware.cpu_fastest_core_mhz</th>\n", + " <th>hardware.cpu_model</th>\n", + " <th>hardware.cpu_threads</th>\n", + " <th>targets.test-instr.singlecore.execs_per_sec</th>\n", + " <th>...</th>\n", + " <th>targets.test-instr.singlecore.fuzzers_used</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.execs_per_sec</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.execs_total</th>\n", + " <th>targets.test-instr-persist-shmem.singlecore.fuzzers_used</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.execs_per_sec</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.execs_total</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.fuzzers_used</th>\n", + " <th>targets.test-instr.multicore.execs_per_sec</th>\n", + " <th>targets.test-instr.multicore.execs_total</th>\n", + " <th>targets.test-instr.multicore.fuzzers_used</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 16.0.6</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>4788.770</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>9845.64</td>\n", + " <td>...</td>\n", + " <td>1.0</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 16.0.6</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>4989.281</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>NaN</td>\n", + " <td>...</td>\n", + " <td>NaN</td>\n", + " <td>125682.73</td>\n", + " <td>1257330.0</td>\n", + " <td>1.0</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 16.0.6</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>4799.415</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>NaN</td>\n", + " <td>...</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>120293.77</td>\n", + " <td>1203058.0</td>\n", + " <td>1.0</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 16.0.6</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>4703.293</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>NaN</td>\n", + " <td>...</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>231429.96</td>\n", + " <td>2314531.0</td>\n", + " <td>2.0</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>i9-9900k, 16GB DDR4-3000, Arch Linux</td>\n", + " <td>clang version 16.0.6</td>\n", + " <td>x86_64-pc-linux-gnu</td>\n", + " <td>4800.375</td>\n", + " <td>Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz</td>\n", + " <td>16</td>\n", + " <td>NaN</td>\n", + " <td>...</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>346759.33</td>\n", + " <td>3468290.0</td>\n", + " <td>3.0</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>5 rows Ć 21 columns</p>\n", + "</div>" + ], + "text/plain": [ + " config.afl_persistent_config config.afl_system_config config.afl_version \\\n", + "0 True True ++4.09a \n", + "1 True True ++4.09a \n", + "2 True True ++4.09a \n", + "3 True True ++4.09a \n", + "4 True True ++4.09a \n", + "\n", + " config.comment config.compiler \\\n", + "0 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n", + "1 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n", + "2 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n", + "3 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n", + "4 i9-9900k, 16GB DDR4-3000, Arch Linux clang version 16.0.6 \n", + "\n", + " config.target_arch hardware.cpu_fastest_core_mhz \\\n", + "0 x86_64-pc-linux-gnu 4788.770 \n", + "1 x86_64-pc-linux-gnu 4989.281 \n", + "2 x86_64-pc-linux-gnu 4799.415 \n", + "3 x86_64-pc-linux-gnu 4703.293 \n", + "4 x86_64-pc-linux-gnu 4800.375 \n", + "\n", + " hardware.cpu_model hardware.cpu_threads \\\n", + "0 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "1 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "2 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "3 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "4 Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz 16 \n", + "\n", + " targets.test-instr.singlecore.execs_per_sec ... \\\n", + "0 9845.64 ... \n", + "1 NaN ... \n", + "2 NaN ... \n", + "3 NaN ... \n", + "4 NaN ... \n", + "\n", + " targets.test-instr.singlecore.fuzzers_used \\\n", + "0 1.0 \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.execs_per_sec \\\n", + "0 NaN \n", + "1 125682.73 \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.execs_total \\\n", + "0 NaN \n", + "1 1257330.0 \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.singlecore.fuzzers_used \\\n", + "0 NaN \n", + "1 1.0 \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr-persist-shmem.multicore.execs_per_sec \\\n", + "0 NaN \n", + "1 NaN \n", + "2 120293.77 \n", + "3 231429.96 \n", + "4 346759.33 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.execs_total \\\n", + "0 NaN \n", + "1 NaN \n", + "2 1203058.0 \n", + "3 2314531.0 \n", + "4 3468290.0 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.fuzzers_used \\\n", + "0 NaN \n", + "1 NaN \n", + "2 1.0 \n", + "3 2.0 \n", + "4 3.0 \n", + "\n", + " targets.test-instr.multicore.execs_per_sec \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr.multicore.execs_total \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " targets.test-instr.multicore.fuzzers_used \n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 144, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "df = pd.json_normalize(json_lines)\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Graph prep\n", + "\n", + "We're looking for a line graph showing lines for each fuzz target, in both singlecore and multicore modes, in each config setting -- where the x-axis is number of cores, and the y-axis is execs_per_sec.\n", + "\n", + "First, a quick check that the number of rows matched what we'd intuitively expect:" + ] + }, + { + "cell_type": "code", + "execution_count": 145, + "metadata": {}, + "outputs": [], + "source": [ + "i7 = df.query(\"`config.comment` == 'i9-9900k, 16GB DDR4-3000, Arch Linux'\")\n", + "assert len(i7) == 185" + ] + }, + { + "cell_type": "code", + "execution_count": 146, + "metadata": {}, + "outputs": [], + "source": [ + "def build_graphdf_from_query(query: pd.DataFrame):\n", + " \"\"\"Build a table suitable for graphing from a subset of the dataframe.\"\"\"\n", + " graphdata = []\n", + " max_fuzzers = int(query[[\"targets.test-instr-persist-shmem.multicore.fuzzers_used\", \"targets.test-instr.multicore.fuzzers_used\"]].max(axis=1).max(axis=0))\n", + " for _, row in query.iterrows():\n", + " for target in [\"test-instr-persist-shmem\", \"test-instr\"]:\n", + " for mode in [\"multicore\", \"singlecore\"]:\n", + " label = \"\"\n", + " if not row[f\"targets.{target}.{mode}.execs_per_sec\"] > 0:\n", + " continue\n", + " execs_per_sec = row[f\"targets.{target}.{mode}.execs_per_sec\"]\n", + " parallel_fuzzers = row[f\"targets.{target}.{mode}.fuzzers_used\"]\n", + " afl_persistent_config = row[\"config.afl_persistent_config\"]\n", + " afl_system_config = row[\"config.afl_system_config\"]\n", + " if target == \"test-instr-persist-shmem\":\n", + " label += \"shmem\"\n", + " else:\n", + " label += \"base\"\n", + " if mode == \"multicore\":\n", + " label += \"-multicore\"\n", + " else:\n", + " label += \"-singlecore\"\n", + " if afl_persistent_config:\n", + " label += \"+persist-conf\"\n", + " if afl_system_config:\n", + " label += \"+system-conf\"\n", + " \n", + " if label == \"shmem-multicore+persist-conf+system-conf\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory + kernel config\"})\n", + " if label == \"shmem-multicore\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Persistent mode/shared memory without kernel config\"})\n", + " if label == \"base-multicore+persist-conf+system-conf\":\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": parallel_fuzzers, \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Multicore: Non-persistent mode + kernel config\"})\n", + " if label == \"shmem-singlecore+persist-conf+system-conf\":\n", + " for i in range(1, max_fuzzers + 1):\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Persistent mode/shared memory + kernel config\"})\n", + " if label == \"base-singlecore+persist-conf+system-conf\":\n", + " for i in range(1, max_fuzzers + 1):\n", + " graphdata.append({\"execs_per_sec\": execs_per_sec, \"parallel_fuzzers\": float(i), \"afl_persistent_config\": afl_persistent_config, \"afl_system_config\": afl_system_config, \"label\": \"Singlecore: Non-persistent mode + kernel config\"})\n", + " return pd.DataFrame.from_records(graphdata).sort_values(\"label\", ascending=False)\n", + "\n", + "graphdf = build_graphdf_from_query(i7)" + ] + }, + { + "cell_type": "code", + "execution_count": 147, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "<svg class=\"main-svg\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"1200\" height=\"400\" style=\"\" viewBox=\"0 0 1200 400\"><rect x=\"0\" y=\"0\" width=\"1200\" height=\"400\" style=\"fill: rgb(255, 255, 255); fill-opacity: 1;\"/><defs id=\"defs-9d6f0e\"><g class=\"clips\"><clipPath id=\"clip9d6f0exyplot\" class=\"plotclip\"><rect width=\"707\" height=\"220\"/></clipPath><clipPath class=\"axesclip\" id=\"clip9d6f0ex\"><rect x=\"80\" y=\"0\" width=\"707\" height=\"400\"/></clipPath><clipPath class=\"axesclip\" id=\"clip9d6f0ey\"><rect x=\"0\" y=\"100\" width=\"1200\" height=\"220\"/></clipPath><clipPath class=\"axesclip\" id=\"clip9d6f0exy\"><rect x=\"80\" y=\"100\" width=\"707\" height=\"220\"/></clipPath></g><g class=\"gradients\"/><g class=\"patterns\"/></defs><g class=\"bglayer\"><rect class=\"bg\" x=\"80\" y=\"100\" width=\"707\" height=\"220\" style=\"fill: rgb(229, 236, 246); fill-opacity: 1; stroke-width: 0;\"/></g><g class=\"layer-below\"><g class=\"imagelayer\"/><g class=\"shapelayer\"/></g><g class=\"cartesianlayer\"><g class=\"subplot xy\"><g class=\"layer-subplot\"><g class=\"shapelayer\"/><g class=\"imagelayer\"/></g><g class=\"minor-gridlayer\"><g class=\"x\"/><g class=\"y\"/></g><g class=\"gridlayer\"><g class=\"x\"><path class=\"xgrid crisp\" transform=\"translate(100.2,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(120.4,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(140.6,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(160.8,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(181,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(201.2,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(221.4,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(241.6,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(261.8,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(282,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(302.2,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(322.4,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(342.6,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(362.8,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(383,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(403.2,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(423.4,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(443.6,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(463.8,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(484,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(504.2,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(524.4,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(544.6,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(564.8,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(585,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(605.2,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(625.4,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(645.6,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(665.8,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(686,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(706.2,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(726.4,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(746.6,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(766.8,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/></g><g class=\"y\"><path class=\"ygrid crisp\" transform=\"translate(0,309)\" d=\"M80,0h707\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,269.4)\" d=\"M80,0h707\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,229.8)\" d=\"M80,0h707\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,190.2)\" d=\"M80,0h707\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,150.6)\" d=\"M80,0h707\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,111)\" d=\"M80,0h707\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/></g></g><g class=\"zerolinelayer\"><path class=\"yzl zl crisp\" transform=\"translate(0,310.68)\" d=\"M80,0h707\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 2px;\"/></g><path class=\"xlines-below\"/><path class=\"ylines-below\"/><g class=\"overlines-below\"/><g class=\"xaxislayer-below\"/><g class=\"yaxislayer-below\"/><g class=\"overaxes-below\"/><g class=\"plot\" transform=\"translate(80,100)\" clip-path=\"url(#clip9d6f0exyplot)\"><g class=\"scatterlayer mlayer\"><g class=\"trace scatter traceff0d16\" style=\"stroke-miterlimit: 2; opacity: 1;\"><g class=\"fills\"/><g class=\"errorbars\"/><g class=\"lines\"><path class=\"js-line\" d=\"M0,208.85L80.8,202.77L101,201.36L141.4,198.59L161.6,198L323.2,193.39L343.4,193.43L464.6,193.88L484.8,193.92L707,194.3\" style=\"vector-effect: none; fill: none; stroke: rgb(99, 110, 250); stroke-opacity: 1; stroke-width: 2px; opacity: 1;\"/></g><g class=\"points\"/><g class=\"text\"/></g><g class=\"trace scatter tracef305d6\" style=\"stroke-miterlimit: 2; opacity: 1;\"><g class=\"fills\"/><g class=\"errorbars\"/><g class=\"lines\"><path class=\"js-line\" d=\"M0,190.13L101,94.86L121.2,77L141.4,58.03L161.6,47.37L202,37L222.2,30.79L242.4,25.2L262.6,20.96L303,11L323.2,16.78L343.4,12.45L363.6,13.25L383.8,14.21L404,16.48L424.2,20.66L444.4,15.41L464.6,15.27L484.8,22.42L505,19.96L525.2,22.91L545.4,20.22L565.6,20.92L585.8,29.86L606,19.39L626.2,19.56L646.4,23.9L666.6,21.39L686.8,32.77L707,24.08\" style=\"vector-effect: none; fill: none; stroke: rgb(239, 85, 59); stroke-opacity: 1; stroke-width: 2px; opacity: 1;\"/></g><g class=\"points\"/><g class=\"text\"/></g><g class=\"trace scatter tracef64091\" style=\"stroke-miterlimit: 2; opacity: 1;\"><g class=\"fills\"/><g class=\"errorbars\"/><g class=\"lines\"><path class=\"js-line\" d=\"M0,195.2L40.4,165.81L60.6,152.75L141.4,97.63L161.6,93.7L303,66.53L323.2,66.65L383.8,65.9L404,66.5L444.4,66.49L464.6,67.67L505,67.82L525.2,69.28L565.6,69.56L585.8,70.99L626.2,71.86L646.4,73.11L666.6,71.31L686.8,71.91L707,71.07\" style=\"vector-effect: none; fill: none; stroke: rgb(0, 204, 150); stroke-opacity: 1; stroke-width: 2px; opacity: 1;\"/></g><g class=\"points\"/><g class=\"text\"/></g><g class=\"trace scatter trace642013\" style=\"stroke-miterlimit: 2; opacity: 1;\"><g class=\"fills\"/><g class=\"errorbars\"/><g class=\"lines\"><path class=\"js-line\" d=\"M0,209L707,209\" style=\"vector-effect: none; fill: none; stroke: rgb(171, 99, 250); stroke-opacity: 1; stroke-width: 2px; opacity: 1;\"/></g><g class=\"points\"/><g class=\"text\"/></g><g class=\"trace scatter traceff0076\" style=\"stroke-miterlimit: 2; opacity: 1;\"><g class=\"fills\"/><g class=\"errorbars\"/><g class=\"lines\"><path class=\"js-line\" d=\"M0,189.21L707,189.21\" style=\"vector-effect: none; fill: none; stroke: rgb(255, 161, 90); stroke-opacity: 1; stroke-width: 2px; opacity: 1;\"/></g><g class=\"points\"/><g class=\"text\"/></g></g></g><g class=\"overplot\"/><path class=\"xlines-above crisp\" d=\"M0,0\" style=\"fill: none;\"/><path class=\"ylines-above crisp\" d=\"M0,0\" style=\"fill: none;\"/><g class=\"overlines-above\"/><g class=\"xaxislayer-above\"><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" transform=\"translate(80,0)\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\">1</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(100.2,0)\">2</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(120.4,0)\">3</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(140.6,0)\">4</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(160.8,0)\">5</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(181,0)\">6</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(201.2,0)\">7</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(221.4,0)\">8</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(241.6,0)\">9</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(261.8,0)\">10</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(282,0)\">11</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(302.2,0)\">12</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(322.4,0)\">13</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(342.6,0)\">14</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(362.8,0)\">15</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(383,0)\">16</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(403.2,0)\">17</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(423.4,0)\">18</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(443.6,0)\">19</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(463.8,0)\">20</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(484,0)\">21</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(504.2,0)\">22</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(524.4,0)\">23</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(544.6,0)\">24</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(564.8,0)\">25</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(585,0)\">26</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(605.2,0)\">27</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(625.4,0)\">28</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(645.6,0)\">29</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(665.8,0)\">30</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(686,0)\">31</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(706.2,0)\">32</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(726.4,0)\">33</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(746.6,0)\">34</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(766.8,0)\">35</text></g><g class=\"xtick\"><text text-anchor=\"middle\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(787,0)\">36</text></g></g><g class=\"yaxislayer-above\"><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" transform=\"translate(0,309)\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\">1x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,269.4)\">25x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,229.8)\">48x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,190.2)\">72x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,150.6)\">95x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,111)\">119x</text></g></g><g class=\"overaxes-above\"/></g></g><g class=\"polarlayer\"/><g class=\"smithlayer\"/><g class=\"ternarylayer\"/><g class=\"geolayer\"/><g class=\"funnelarealayer\"/><g class=\"pielayer\"/><g class=\"iciclelayer\"/><g class=\"treemaplayer\"/><g class=\"sunburstlayer\"/><g class=\"glimages\"/><defs id=\"topdefs-9d6f0e\"><g class=\"clips\"/><clipPath id=\"legend9d6f0e\"><rect width=\"387\" height=\"124\" x=\"0\" y=\"0\"/></clipPath></defs><g class=\"layer-above\"><g class=\"imagelayer\"/><g class=\"shapelayer\"/></g><g class=\"infolayer\"><g class=\"legend\" pointer-events=\"all\" transform=\"translate(801.14,100)\"><rect class=\"bg\" shape-rendering=\"crispEdges\" width=\"387\" height=\"124\" x=\"0\" y=\"0\" style=\"stroke: rgb(68, 68, 68); stroke-opacity: 1; fill: rgb(255, 255, 255); fill-opacity: 1; stroke-width: 0px;\"/><g class=\"scrollbox\" transform=\"\" clip-path=\"url(#legend9d6f0e)\"><text class=\"legendtitletext\" text-anchor=\"start\" x=\"2\" y=\"18.2\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 14px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre;\">Configuration</text><g class=\"groups\" transform=\"\"><g class=\"traces\" transform=\"translate(0,32.7)\" style=\"opacity: 1;\"><text class=\"legendtext\" text-anchor=\"start\" x=\"40\" y=\"4.680000000000001\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre;\">Multicore: Non-persistent mode + kernel config</text><g class=\"layers\" style=\"opacity: 1;\"><g class=\"legendfill\"/><g class=\"legendlines\"><path class=\"js-line\" d=\"M5,0h30\" style=\"fill: none; stroke: rgb(99, 110, 250); stroke-opacity: 1; stroke-width: 2px;\"/></g><g class=\"legendsymbols\"><g class=\"legendpoints\"/></g></g><rect class=\"legendtoggle\" x=\"0\" y=\"-9.5\" width=\"381.484375\" height=\"19\" style=\"fill: rgb(0, 0, 0); fill-opacity: 0;\"/></g></g><g class=\"groups\" transform=\"\"><g class=\"traces\" transform=\"translate(0,51.7)\" style=\"opacity: 1;\"><text class=\"legendtext\" text-anchor=\"start\" x=\"40\" y=\"4.680000000000001\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre;\">Multicore: Persistent mode/shared memory + kernel config</text><g class=\"layers\" style=\"opacity: 1;\"><g class=\"legendfill\"/><g class=\"legendlines\"><path class=\"js-line\" d=\"M5,0h30\" style=\"fill: none; stroke: rgb(239, 85, 59); stroke-opacity: 1; stroke-width: 2px;\"/></g><g class=\"legendsymbols\"><g class=\"legendpoints\"/></g></g><rect class=\"legendtoggle\" x=\"0\" y=\"-9.5\" width=\"381.484375\" height=\"19\" style=\"fill: rgb(0, 0, 0); fill-opacity: 0;\"/></g></g><g class=\"groups\" transform=\"\"><g class=\"traces\" transform=\"translate(0,70.7)\" style=\"opacity: 1;\"><text class=\"legendtext\" text-anchor=\"start\" x=\"40\" y=\"4.680000000000001\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre;\">Multicore: Persistent mode/shared memory without kernel config</text><g class=\"layers\" style=\"opacity: 1;\"><g class=\"legendfill\"/><g class=\"legendlines\"><path class=\"js-line\" d=\"M5,0h30\" style=\"fill: none; stroke: rgb(0, 204, 150); stroke-opacity: 1; stroke-width: 2px;\"/></g><g class=\"legendsymbols\"><g class=\"legendpoints\"/></g></g><rect class=\"legendtoggle\" x=\"0\" y=\"-9.5\" width=\"381.484375\" height=\"19\" style=\"fill: rgb(0, 0, 0); fill-opacity: 0;\"/></g></g><g class=\"groups\" transform=\"\"><g class=\"traces\" transform=\"translate(0,89.7)\" style=\"opacity: 1;\"><text class=\"legendtext\" text-anchor=\"start\" x=\"40\" y=\"4.680000000000001\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre;\">Singlecore: Non-persistent mode + kernel config</text><g class=\"layers\" style=\"opacity: 1;\"><g class=\"legendfill\"/><g class=\"legendlines\"><path class=\"js-line\" d=\"M5,0h30\" style=\"fill: none; stroke: rgb(171, 99, 250); stroke-opacity: 1; stroke-width: 2px;\"/></g><g class=\"legendsymbols\"><g class=\"legendpoints\"/></g></g><rect class=\"legendtoggle\" x=\"0\" y=\"-9.5\" width=\"381.484375\" height=\"19\" style=\"fill: rgb(0, 0, 0); fill-opacity: 0;\"/></g></g><g class=\"groups\" transform=\"\"><g class=\"traces\" transform=\"translate(0,108.7)\" style=\"opacity: 1;\"><text class=\"legendtext\" text-anchor=\"start\" x=\"40\" y=\"4.680000000000001\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre;\">Singlecore: Persistent mode/shared memory + kernel config</text><g class=\"layers\" style=\"opacity: 1;\"><g class=\"legendfill\"/><g class=\"legendlines\"><path class=\"js-line\" d=\"M5,0h30\" style=\"fill: none; stroke: rgb(255, 161, 90); stroke-opacity: 1; stroke-width: 2px;\"/></g><g class=\"legendsymbols\"><g class=\"legendpoints\"/></g></g><rect class=\"legendtoggle\" x=\"0\" y=\"-9.5\" width=\"381.484375\" height=\"19\" style=\"fill: rgb(0, 0, 0); fill-opacity: 0;\"/></g></g></g><rect class=\"scrollbar\" rx=\"20\" ry=\"3\" width=\"0\" height=\"0\" x=\"0\" y=\"0\" style=\"fill: rgb(128, 139, 164); fill-opacity: 1;\"/></g><g class=\"g-gtitle\"><text class=\"gtitle\" x=\"60\" y=\"50\" text-anchor=\"start\" dy=\"0em\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 17px; fill: rgb(42, 63, 95); opacity: 1; font-weight: normal; white-space: pre;\">Fuzzer performance</text></g><g class=\"g-xtitle\"><text class=\"xtitle\" x=\"433.5\" y=\"360.3\" text-anchor=\"middle\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 14px; fill: rgb(42, 63, 95); opacity: 1; font-weight: normal; white-space: pre;\">Number of parallel fuzzers</text></g><g class=\"g-ytitle\"><text class=\"ytitle\" transform=\"rotate(-90,29.559375000000003,210)\" x=\"29.559375000000003\" y=\"210\" text-anchor=\"middle\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 14px; fill: rgb(42, 63, 95); opacity: 1; font-weight: normal; white-space: pre;\">Fuzz target executions per second</text></g></g></svg>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import numpy as np\n", + "pd.options.plotting.backend = \"plotly\"\n", + "\n", + "# Right now our table has absolute values of execs per sec, but it's more useful\n", + "# to show relative perf (vs 1.0x baseline)\n", + "pivotdf = graphdf.pivot(index=\"parallel_fuzzers\", columns=\"label\", values=\"execs_per_sec\")\n", + "fig = pivotdf.plot(\n", + " title=\"Fuzzer performance\",\n", + " labels={\n", + " \"label\": \"Configuration\",\n", + " \"parallel_fuzzers\": \"Number of parallel fuzzers\",\n", + " \"value\": \"Fuzz target executions per second\"\n", + " }\n", + ")\n", + "\n", + "# Compute tick values and their labels for the primary Y-axis\n", + "tickvals = np.linspace(graphdf['execs_per_sec'].min(), graphdf['execs_per_sec'].max(), 6)\n", + "ticktext = [f\"{val:.0f}x\" for val in tickvals / graphdf['execs_per_sec'].min()]\n", + "# Update the primary Y-axis with custom tick labels\n", + "fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n", + "fig.update_xaxes(tickvals=list(range(1,36+1)))\n", + "fig.update_layout(width=1200, height=400)\n", + "fig.show(\"svg\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's what the table that produced this graph looks like:" + ] + }, + { + "cell_type": "code", + "execution_count": 148, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th>label</th>\n", + " <th>Multicore: Non-persistent mode + kernel config</th>\n", + " <th>Multicore: Persistent mode/shared memory + kernel config</th>\n", + " <th>Multicore: Persistent mode/shared memory without kernel config</th>\n", + " <th>Singlecore: Non-persistent mode + kernel config</th>\n", + " <th>Singlecore: Persistent mode/shared memory + kernel config</th>\n", + " </tr>\n", + " <tr>\n", + " <th>parallel_fuzzers</th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>1.0</th>\n", + " <td>10714.79</td>\n", + " <td>120293.77</td>\n", + " <td>90641.62</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2.0</th>\n", + " <td>20493.07</td>\n", + " <td>231429.96</td>\n", + " <td>178184.19</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3.0</th>\n", + " <td>29660.06</td>\n", + " <td>346759.33</td>\n", + " <td>262652.86</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4.0</th>\n", + " <td>37875.57</td>\n", + " <td>455340.06</td>\n", + " <td>339119.32</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>5.0</th>\n", + " <td>46326.75</td>\n", + " <td>568405.15</td>\n", + " <td>420239.94</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>6.0</th>\n", + " <td>54595.48</td>\n", + " <td>678030.96</td>\n", + " <td>498062.02</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>7.0</th>\n", + " <td>62720.98</td>\n", + " <td>782585.04</td>\n", + " <td>578495.44</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>8.0</th>\n", + " <td>70777.99</td>\n", + " <td>893618.35</td>\n", + " <td>661836.22</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>9.0</th>\n", + " <td>74236.02</td>\n", + " <td>956026.15</td>\n", + " <td>684808.49</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>10.0</th>\n", + " <td>78134.94</td>\n", + " <td>984942.13</td>\n", + " <td>707094.65</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>11.0</th>\n", + " <td>81886.33</td>\n", + " <td>1016758.62</td>\n", + " <td>732106.17</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>12.0</th>\n", + " <td>85923.44</td>\n", + " <td>1053087.90</td>\n", + " <td>752910.17</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>13.0</th>\n", + " <td>89696.95</td>\n", + " <td>1085797.87</td>\n", + " <td>776179.85</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>14.0</th>\n", + " <td>93540.52</td>\n", + " <td>1110640.20</td>\n", + " <td>797520.58</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>15.0</th>\n", + " <td>97641.51</td>\n", + " <td>1138984.22</td>\n", + " <td>822235.41</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>16.0</th>\n", + " <td>101692.65</td>\n", + " <td>1168943.19</td>\n", + " <td>843897.51</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>17.0</th>\n", + " <td>101236.75</td>\n", + " <td>1135093.91</td>\n", + " <td>843177.15</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>18.0</th>\n", + " <td>101006.28</td>\n", + " <td>1160430.45</td>\n", + " <td>844779.09</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>19.0</th>\n", + " <td>99952.26</td>\n", + " <td>1155769.97</td>\n", + " <td>846060.74</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>20.0</th>\n", + " <td>99798.64</td>\n", + " <td>1150156.26</td>\n", + " <td>847556.23</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>21.0</th>\n", + " <td>99018.86</td>\n", + " <td>1136873.58</td>\n", + " <td>844022.97</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>22.0</th>\n", + " <td>98600.87</td>\n", + " <td>1112404.25</td>\n", + " <td>845818.70</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>23.0</th>\n", + " <td>98634.02</td>\n", + " <td>1143131.72</td>\n", + " <td>844118.27</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>24.0</th>\n", + " <td>98352.90</td>\n", + " <td>1143931.38</td>\n", + " <td>837189.02</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>25.0</th>\n", + " <td>98118.63</td>\n", + " <td>1102090.61</td>\n", + " <td>834712.31</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>26.0</th>\n", + " <td>97752.45</td>\n", + " <td>1116518.70</td>\n", + " <td>836344.12</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>27.0</th>\n", + " <td>97864.07</td>\n", + " <td>1099224.19</td>\n", + " <td>827784.91</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>28.0</th>\n", + " <td>97821.80</td>\n", + " <td>1114945.37</td>\n", + " <td>828641.27</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>29.0</th>\n", + " <td>97564.87</td>\n", + " <td>1110889.91</td>\n", + " <td>826123.67</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>30.0</th>\n", + " <td>98508.10</td>\n", + " <td>1058548.28</td>\n", + " <td>817765.77</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>31.0</th>\n", + " <td>98238.96</td>\n", + " <td>1119804.85</td>\n", + " <td>816556.66</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>32.0</th>\n", + " <td>98363.93</td>\n", + " <td>1118828.99</td>\n", + " <td>812661.77</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>33.0</th>\n", + " <td>96758.69</td>\n", + " <td>1093426.61</td>\n", + " <td>805352.16</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>34.0</th>\n", + " <td>96327.00</td>\n", + " <td>1108123.59</td>\n", + " <td>815888.26</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>35.0</th>\n", + " <td>95913.98</td>\n", + " <td>1041486.52</td>\n", + " <td>812348.56</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " <tr>\n", + " <th>36.0</th>\n", + " <td>95871.39</td>\n", + " <td>1092395.61</td>\n", + " <td>817278.03</td>\n", + " <td>9845.64</td>\n", + " <td>125682.73</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + "label Multicore: Non-persistent mode + kernel config \\\n", + "parallel_fuzzers \n", + "1.0 10714.79 \n", + "2.0 20493.07 \n", + "3.0 29660.06 \n", + "4.0 37875.57 \n", + "5.0 46326.75 \n", + "6.0 54595.48 \n", + "7.0 62720.98 \n", + "8.0 70777.99 \n", + "9.0 74236.02 \n", + "10.0 78134.94 \n", + "11.0 81886.33 \n", + "12.0 85923.44 \n", + "13.0 89696.95 \n", + "14.0 93540.52 \n", + "15.0 97641.51 \n", + "16.0 101692.65 \n", + "17.0 101236.75 \n", + "18.0 101006.28 \n", + "19.0 99952.26 \n", + "20.0 99798.64 \n", + "21.0 99018.86 \n", + "22.0 98600.87 \n", + "23.0 98634.02 \n", + "24.0 98352.90 \n", + "25.0 98118.63 \n", + "26.0 97752.45 \n", + "27.0 97864.07 \n", + "28.0 97821.80 \n", + "29.0 97564.87 \n", + "30.0 98508.10 \n", + "31.0 98238.96 \n", + "32.0 98363.93 \n", + "33.0 96758.69 \n", + "34.0 96327.00 \n", + "35.0 95913.98 \n", + "36.0 95871.39 \n", + "\n", + "label Multicore: Persistent mode/shared memory + kernel config \\\n", + "parallel_fuzzers \n", + "1.0 120293.77 \n", + "2.0 231429.96 \n", + "3.0 346759.33 \n", + "4.0 455340.06 \n", + "5.0 568405.15 \n", + "6.0 678030.96 \n", + "7.0 782585.04 \n", + "8.0 893618.35 \n", + "9.0 956026.15 \n", + "10.0 984942.13 \n", + "11.0 1016758.62 \n", + "12.0 1053087.90 \n", + "13.0 1085797.87 \n", + "14.0 1110640.20 \n", + "15.0 1138984.22 \n", + "16.0 1168943.19 \n", + "17.0 1135093.91 \n", + "18.0 1160430.45 \n", + "19.0 1155769.97 \n", + "20.0 1150156.26 \n", + "21.0 1136873.58 \n", + "22.0 1112404.25 \n", + "23.0 1143131.72 \n", + "24.0 1143931.38 \n", + "25.0 1102090.61 \n", + "26.0 1116518.70 \n", + "27.0 1099224.19 \n", + "28.0 1114945.37 \n", + "29.0 1110889.91 \n", + "30.0 1058548.28 \n", + "31.0 1119804.85 \n", + "32.0 1118828.99 \n", + "33.0 1093426.61 \n", + "34.0 1108123.59 \n", + "35.0 1041486.52 \n", + "36.0 1092395.61 \n", + "\n", + "label Multicore: Persistent mode/shared memory without kernel config \\\n", + "parallel_fuzzers \n", + "1.0 90641.62 \n", + "2.0 178184.19 \n", + "3.0 262652.86 \n", + "4.0 339119.32 \n", + "5.0 420239.94 \n", + "6.0 498062.02 \n", + "7.0 578495.44 \n", + "8.0 661836.22 \n", + "9.0 684808.49 \n", + "10.0 707094.65 \n", + "11.0 732106.17 \n", + "12.0 752910.17 \n", + "13.0 776179.85 \n", + "14.0 797520.58 \n", + "15.0 822235.41 \n", + "16.0 843897.51 \n", + "17.0 843177.15 \n", + "18.0 844779.09 \n", + "19.0 846060.74 \n", + "20.0 847556.23 \n", + "21.0 844022.97 \n", + "22.0 845818.70 \n", + "23.0 844118.27 \n", + "24.0 837189.02 \n", + "25.0 834712.31 \n", + "26.0 836344.12 \n", + "27.0 827784.91 \n", + "28.0 828641.27 \n", + "29.0 826123.67 \n", + "30.0 817765.77 \n", + "31.0 816556.66 \n", + "32.0 812661.77 \n", + "33.0 805352.16 \n", + "34.0 815888.26 \n", + "35.0 812348.56 \n", + "36.0 817278.03 \n", + "\n", + "label Singlecore: Non-persistent mode + kernel config \\\n", + "parallel_fuzzers \n", + "1.0 9845.64 \n", + "2.0 9845.64 \n", + "3.0 9845.64 \n", + "4.0 9845.64 \n", + "5.0 9845.64 \n", + "6.0 9845.64 \n", + "7.0 9845.64 \n", + "8.0 9845.64 \n", + "9.0 9845.64 \n", + "10.0 9845.64 \n", + "11.0 9845.64 \n", + "12.0 9845.64 \n", + "13.0 9845.64 \n", + "14.0 9845.64 \n", + "15.0 9845.64 \n", + "16.0 9845.64 \n", + "17.0 9845.64 \n", + "18.0 9845.64 \n", + "19.0 9845.64 \n", + "20.0 9845.64 \n", + "21.0 9845.64 \n", + "22.0 9845.64 \n", + "23.0 9845.64 \n", + "24.0 9845.64 \n", + "25.0 9845.64 \n", + "26.0 9845.64 \n", + "27.0 9845.64 \n", + "28.0 9845.64 \n", + "29.0 9845.64 \n", + "30.0 9845.64 \n", + "31.0 9845.64 \n", + "32.0 9845.64 \n", + "33.0 9845.64 \n", + "34.0 9845.64 \n", + "35.0 9845.64 \n", + "36.0 9845.64 \n", + "\n", + "label Singlecore: Persistent mode/shared memory + kernel config \n", + "parallel_fuzzers \n", + "1.0 125682.73 \n", + "2.0 125682.73 \n", + "3.0 125682.73 \n", + "4.0 125682.73 \n", + "5.0 125682.73 \n", + "6.0 125682.73 \n", + "7.0 125682.73 \n", + "8.0 125682.73 \n", + "9.0 125682.73 \n", + "10.0 125682.73 \n", + "11.0 125682.73 \n", + "12.0 125682.73 \n", + "13.0 125682.73 \n", + "14.0 125682.73 \n", + "15.0 125682.73 \n", + "16.0 125682.73 \n", + "17.0 125682.73 \n", + "18.0 125682.73 \n", + "19.0 125682.73 \n", + "20.0 125682.73 \n", + "21.0 125682.73 \n", + "22.0 125682.73 \n", + "23.0 125682.73 \n", + "24.0 125682.73 \n", + "25.0 125682.73 \n", + "26.0 125682.73 \n", + "27.0 125682.73 \n", + "28.0 125682.73 \n", + "29.0 125682.73 \n", + "30.0 125682.73 \n", + "31.0 125682.73 \n", + "32.0 125682.73 \n", + "33.0 125682.73 \n", + "34.0 125682.73 \n", + "35.0 125682.73 \n", + "36.0 125682.73 " + ] + }, + "execution_count": 148, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pivotdf" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can totally ignore the code cell directly below (unless you're curious). It's just preparing Markdown for the block below it to render. Jupyter Notebooks aren't able to use code variables inside Markdown blocks, so I have to do this instead." + ] + }, + { + "cell_type": "code", + "execution_count": 149, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "\n", + "### Line graph analysis\n", + "Here are a few things that jump out from the graph above. Let's start at the bottom of the graph.\n", + "\n", + "#### test-instr vs. test-instr-persist-shmem\n", + "\n", + "This graph is scaled so that the single-core, non-persistent-mode performance (9845 execs per second) is\n", + "represented as **1.0x**. If you build and run a fuzzer without creating a persistent mode harness for it, and without running fuzzers in parallel, this is the performance\n", + "you get on this machine.\n", + "\n", + "#### Multicore test-instr\n", + "\n", + "By running as many parallel fuzzers are there are CPU threads, we can reach 101692 execs per second, which is **10.3x** that base speed.\n", + "\n", + "#### Persistent mode + shared memory\n", + "\n", + "##### Singlecore\n", + "\n", + "By modifying the harness to use persistent mode with shared memory as described [here](https://github.com/AFLplusplus/AFLplusplus/blob/stable/instrumentation/README.persistent_mode.md#4-persistent-mode),\n", + "we end up with **12.8x** base speed. So -- perhaps counter-intuively -- if you have a choice between switching to using multiple cores or rewriting\n", + "the harness to use persistent mode on a single core, it is better (at least on this machine) to use persistent mode on a single core, than to use non-persistent mode on all cores.\n", + "\n", + "##### Multicore\n", + "\n", + "By scaling up that persistent mode with shared memory harness across cores, and with kernel mitigations still turned on (see next section), we get to\n", + "**86.1x** base speed.\n", + "\n", + "#### Kernel config\n", + "\n", + "By \"kernel config\", I'm referring to booting the Linux kernel with `mitigations=off`, which is a meta-parameter for disabling *all* hardware vulnerability meltdowns (such as Spectre,\n", + "Meltdown, Retbleed, etc) introduced in Linux v5.2. Disabling these results in a `execs_per_sec` increase of 321386 execs -- the difference between\n", + "118.7x (mitigations off) and 86.1x (mitigations on) base speed. Turning on mitigations\n", + "reduced the overall performance by 27%!\n", + "\n", + "One way to think about this is that the mitigations turn this 16-thread CPU into a 7-thread CPU, since the number of execs reached with 16 threads and mitigations on is around the same\n", + "number of execs reached with 7 threads and mitigations off.\n", + "\n", + "Or if we want to think in terms of cores, then the average number of execs gained per core in the initial eight is 110474 execs per sec, but the loss due to\n", + "mitigations is 321386 execs per sec, which is the averaged performance of 2.9 cores.\n", + "\n", + "With kernel mitigations turned off, we reach our highest available execs_per_sec speed on this machine, which is **118.7x** higher\n", + "than where we started from.\n", + "\n", + "#### How many parallel fuzzers should we use on this machine?\n", + "\n", + "* Using >16 is worse than using 16. Makes sense.\n", + "* So, we should use the number of CPUs in /proc/cpuinfo (threads) to get the best performance. But if we did halve the number of\n", + " fuzzers, we would surprisingly only lose 23%\n", + " of performance. This could be a good tradeoff in terms of cost.\n" + ], + "text/plain": [ + "<IPython.core.display.Markdown object>" + ] + }, + "execution_count": 149, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# (You can ignore reading this code cell.)\n", + "from IPython.display import Markdown as md\n", + "singlecore_base_execs = pivotdf.iloc[0][\"Singlecore: Non-persistent mode + kernel config\"]\n", + "singlecore_persist_execs = pivotdf.iloc[0][\"Singlecore: Persistent mode/shared memory + kernel config\"]\n", + "multicore_fuzzers_max_execs = int(pivotdf[\"Multicore: Persistent mode/shared memory + kernel config\"].idxmax())\n", + "multicore_base_max_execs = pivotdf[\"Multicore: Non-persistent mode + kernel config\"].max()\n", + "factor_for_execs = lambda execs: round(execs / singlecore_base_execs, 1)\n", + "\n", + "multicore_persistent_without_mitigations_label = \"Multicore: Persistent mode/shared memory + kernel config\"\n", + "multicore_max_execs_mitigations_off = pivotdf[multicore_persistent_without_mitigations_label].max()\n", + "multicore_max_execs_mitigations_off_only_cores = pivotdf.loc[multicore_fuzzers_max_execs / 2][multicore_persistent_without_mitigations_label]\n", + "multicore_max_execs_mitigations_on = pivotdf[\"Multicore: Persistent mode/shared memory without kernel config\"].max()\n", + "multicore_avg_gain_per_core = pivotdf.loc[pivotdf.index <= 8][\"Multicore: Persistent mode/shared memory + kernel config\"].diff().dropna().mean()\n", + "mitigations_off_increase = int(multicore_max_execs_mitigations_off - multicore_max_execs_mitigations_on)\n", + "\n", + "md(f\"\"\"\n", + "### Line graph analysis\n", + "Here are a few things that jump out from the graph above. Let's start at the bottom of the graph.\n", + "\n", + "#### test-instr vs. test-instr-persist-shmem\n", + "\n", + "This graph is scaled so that the single-core, non-persistent-mode performance ({int(singlecore_base_execs)} execs per second) is\n", + "represented as **1.0x**. If you build and run a fuzzer without creating a persistent mode harness for it, and without running fuzzers in parallel, this is the performance\n", + "you get on this machine.\n", + "\n", + "#### Multicore test-instr\n", + "\n", + "By running as many parallel fuzzers are there are CPU threads, we can reach {int(multicore_base_max_execs)} execs per second, which is **{factor_for_execs(multicore_base_max_execs)}x** that base speed.\n", + "\n", + "#### Persistent mode + shared memory\n", + "\n", + "##### Singlecore\n", + "\n", + "By modifying the harness to use persistent mode with shared memory as described [here](https://github.com/AFLplusplus/AFLplusplus/blob/stable/instrumentation/README.persistent_mode.md#4-persistent-mode),\n", + "we end up with **{factor_for_execs(singlecore_persist_execs)}x** base speed. So -- perhaps counter-intuively -- if you have a choice between switching to using multiple cores or rewriting\n", + "the harness to use persistent mode on a single core, it is better (at least on this machine) to use persistent mode on a single core, than to use non-persistent mode on all cores.\n", + "\n", + "##### Multicore\n", + "\n", + "By scaling up that persistent mode with shared memory harness across cores, and with kernel mitigations still turned on (see next section), we get to\n", + "**{factor_for_execs(multicore_max_execs_mitigations_on)}x** base speed.\n", + "\n", + "#### Kernel config\n", + "\n", + "By \"kernel config\", I'm referring to booting the Linux kernel with `mitigations=off`, which is a meta-parameter for disabling *all* hardware vulnerability meltdowns (such as Spectre,\n", + "Meltdown, Retbleed, etc) introduced in Linux v5.2. Disabling these results in a `execs_per_sec` increase of {mitigations_off_increase} execs -- the difference between\n", + "{factor_for_execs(multicore_max_execs_mitigations_off)}x (mitigations off) and {factor_for_execs(multicore_max_execs_mitigations_on)}x (mitigations on) base speed. Turning on mitigations\n", + "reduced the overall performance by {abs(round(((multicore_max_execs_mitigations_on - multicore_max_execs_mitigations_off) / multicore_max_execs_mitigations_off) * 100))}%!\n", + "\n", + "One way to think about this is that the mitigations turn this 16-thread CPU into a 7-thread CPU, since the number of execs reached with 16 threads and mitigations on is around the same\n", + "number of execs reached with 7 threads and mitigations off.\n", + "\n", + "Or if we want to think in terms of cores, then the average number of execs gained per core in the initial eight is {int(multicore_avg_gain_per_core)} execs per sec, but the loss due to\n", + "mitigations is {mitigations_off_increase} execs per sec, which is the averaged performance of {round(mitigations_off_increase / multicore_avg_gain_per_core, 1)} cores.\n", + "\n", + "With kernel mitigations turned off, we reach our highest available execs_per_sec speed on this machine, which is **{factor_for_execs(multicore_max_execs_mitigations_off)}x** higher\n", + "than where we started from.\n", + "\n", + "#### How many parallel fuzzers should we use on this machine?\n", + "\n", + "* Using >16 is worse than using 16. Makes sense.\n", + "* So, we should use the number of CPUs in /proc/cpuinfo (threads) to get the best performance. But if we did halve the number of\n", + " fuzzers, we would surprisingly only lose {abs(int(((multicore_max_execs_mitigations_off_only_cores - multicore_max_execs_mitigations_off) / multicore_max_execs_mitigations_off) * 100))}%\n", + " of performance. This could be a good tradeoff in terms of cost.\n", + "\"\"\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Example with more cores\n", + "\n", + "While there was some nuance here, the answer was pretty straightforward -- use the number of CPU threads you have access to. What if there were more threads? Here the experiment is repeated on an AWS EC2 \"r6a.48xlarge\" spot instance with 192 vCPUs:" + ] + }, + { + "cell_type": "code", + "execution_count": 150, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>config.afl_persistent_config</th>\n", + " <th>config.afl_system_config</th>\n", + " <th>config.afl_version</th>\n", + " <th>config.comment</th>\n", + " <th>config.compiler</th>\n", + " <th>config.target_arch</th>\n", + " <th>hardware.cpu_fastest_core_mhz</th>\n", + " <th>hardware.cpu_model</th>\n", + " <th>hardware.cpu_threads</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.execs_per_sec</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.execs_total</th>\n", + " <th>targets.test-instr-persist-shmem.multicore.fuzzers_used</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>223</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>AWS EC2 r6a.48xlarge spot instance</td>\n", + " <td>clang version 15.0.7 (Amazon Linux 15.0.7-3.am...</td>\n", + " <td>x86_64-amazon-linux-gnu</td>\n", + " <td>3514.326</td>\n", + " <td>AMD EPYC 7R13 Processor</td>\n", + " <td>192</td>\n", + " <td>119469.35</td>\n", + " <td>1194813.0</td>\n", + " <td>1.0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>224</th>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>++4.09a</td>\n", + " <td>AWS EC2 r6a.48xlarge spot instance</td>\n", + " <td>clang version 15.0.7 (Amazon Linux 15.0.7-3.am...</td>\n", + " <td>x86_64-amazon-linux-gnu</td>\n", + " <td>3599.748</td>\n", + " <td>AMD EPYC 7R13 Processor</td>\n", + " <td>192</td>\n", + " <td>237177.20</td>\n", + " <td>2372250.0</td>\n", + " <td>2.0</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " config.afl_persistent_config config.afl_system_config \\\n", + "223 True True \n", + "224 True True \n", + "\n", + " config.afl_version config.comment \\\n", + "223 ++4.09a AWS EC2 r6a.48xlarge spot instance \n", + "224 ++4.09a AWS EC2 r6a.48xlarge spot instance \n", + "\n", + " config.compiler \\\n", + "223 clang version 15.0.7 (Amazon Linux 15.0.7-3.am... \n", + "224 clang version 15.0.7 (Amazon Linux 15.0.7-3.am... \n", + "\n", + " config.target_arch hardware.cpu_fastest_core_mhz \\\n", + "223 x86_64-amazon-linux-gnu 3514.326 \n", + "224 x86_64-amazon-linux-gnu 3599.748 \n", + "\n", + " hardware.cpu_model hardware.cpu_threads \\\n", + "223 AMD EPYC 7R13 Processor 192 \n", + "224 AMD EPYC 7R13 Processor 192 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.execs_per_sec \\\n", + "223 119469.35 \n", + "224 237177.20 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.execs_total \\\n", + "223 1194813.0 \n", + "224 2372250.0 \n", + "\n", + " targets.test-instr-persist-shmem.multicore.fuzzers_used \n", + "223 1.0 \n", + "224 2.0 " + ] + }, + "execution_count": 150, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r6a = df.query(\"`config.comment` == 'AWS EC2 r6a.48xlarge spot instance'\")\n", + "r6a.head(2).dropna(axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 151, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>execs_per_sec</th>\n", + " <th>parallel_fuzzers</th>\n", + " <th>afl_persistent_config</th>\n", + " <th>afl_system_config</th>\n", + " <th>label</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>119469.35</td>\n", + " <td>1.0</td>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>Multicore: Persistent mode/shared memory + ker...</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>237177.20</td>\n", + " <td>2.0</td>\n", + " <td>True</td>\n", + " <td>True</td>\n", + " <td>Multicore: Persistent mode/shared memory + ker...</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " execs_per_sec parallel_fuzzers afl_persistent_config afl_system_config \\\n", + "0 119469.35 1.0 True True \n", + "1 237177.20 2.0 True True \n", + "\n", + " label \n", + "0 Multicore: Persistent mode/shared memory + ker... \n", + "1 Multicore: Persistent mode/shared memory + ker... " + ] + }, + "execution_count": 151, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r6a_graphdf = build_graphdf_from_query(r6a)\n", + "r6a_graphdf.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 152, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "<svg class=\"main-svg\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"1200\" height=\"400\" style=\"\" viewBox=\"0 0 1200 400\"><rect x=\"0\" y=\"0\" width=\"1200\" height=\"400\" style=\"fill: rgb(255, 255, 255); fill-opacity: 1;\"/><defs id=\"defs-cbb66b\"><g class=\"clips\"><clipPath id=\"clipcbb66bxyplot\" class=\"plotclip\"><rect width=\"737\" height=\"220\"/></clipPath><clipPath class=\"axesclip\" id=\"clipcbb66bx\"><rect x=\"80\" y=\"0\" width=\"737\" height=\"400\"/></clipPath><clipPath class=\"axesclip\" id=\"clipcbb66by\"><rect x=\"0\" y=\"100\" width=\"1200\" height=\"220\"/></clipPath><clipPath class=\"axesclip\" id=\"clipcbb66bxy\"><rect x=\"80\" y=\"100\" width=\"737\" height=\"220\"/></clipPath></g><g class=\"gradients\"/><g class=\"patterns\"/></defs><g class=\"bglayer\"><rect class=\"bg\" x=\"80\" y=\"100\" width=\"737\" height=\"220\" style=\"fill: rgb(229, 236, 246); fill-opacity: 1; stroke-width: 0;\"/></g><g class=\"layer-below\"><g class=\"imagelayer\"/><g class=\"shapelayer\"/></g><g class=\"cartesianlayer\"><g class=\"subplot xy\"><g class=\"layer-subplot\"><g class=\"shapelayer\"/><g class=\"imagelayer\"/></g><g class=\"minor-gridlayer\"><g class=\"x\"/><g class=\"y\"/></g><g class=\"gridlayer\"><g class=\"x\"><path class=\"xgrid crisp\" transform=\"translate(91.58,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(107.01,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(122.45,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(137.88,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(153.31,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(168.75,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(184.18,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(199.62,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(215.05,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(230.49,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(245.92,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(261.36,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(276.78999999999996,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(292.23,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(307.65999999999997,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(323.09000000000003,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(338.53,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(353.96,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(369.4,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(384.83,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(400.27,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(415.7,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(431.14,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(446.57,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(462.01,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(477.44,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(492.87,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(508.31,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(523.74,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(539.1800000000001,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(554.61,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(570.05,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(585.48,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(600.92,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(616.35,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(631.79,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(647.22,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(662.65,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(678.09,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(693.52,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(708.96,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(724.39,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(739.83,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(755.26,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(770.7,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(786.13,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"xgrid crisp\" transform=\"translate(801.57,0)\" d=\"M0,100v220\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/></g><g class=\"y\"><path class=\"ygrid crisp\" transform=\"translate(0,309)\" d=\"M80,0h737\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,269.4)\" d=\"M80,0h737\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,229.8)\" d=\"M80,0h737\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,190.2)\" d=\"M80,0h737\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,150.6)\" d=\"M80,0h737\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/><path class=\"ygrid crisp\" transform=\"translate(0,111)\" d=\"M80,0h737\" style=\"stroke: rgb(255, 255, 255); stroke-opacity: 1; stroke-width: 1px;\"/></g></g><g class=\"zerolinelayer\"/><path class=\"xlines-below\"/><path class=\"ylines-below\"/><g class=\"overlines-below\"/><g class=\"xaxislayer-below\"/><g class=\"yaxislayer-below\"/><g class=\"overaxes-below\"/><g class=\"plot\" transform=\"translate(80,100)\" clip-path=\"url(#clipcbb66bxyplot)\"><g class=\"scatterlayer mlayer\"><g class=\"trace scatter trace9250e0\" style=\"stroke-miterlimit: 2; opacity: 1;\"><g class=\"fills\"/><g class=\"errorbars\"/><g class=\"lines\"><path class=\"js-line\" d=\"M0,209L27.01,102.25L30.87,92.62L38.59,70.35L42.45,56.73L54.02,21.04L57.88,11L61.74,26.13L65.6,32.45L77.17,55.69L81.03,63.3L88.75,77.57L92.61,91.06L100.32,99.97L104.18,101.5L119.62,110.06L123.48,115.54L131.19,117.24L135.05,116.27L138.91,118.19L142.77,119.19L150.49,122.55L154.35,126.03L158.2,124.89L162.06,125.39L165.92,124.75L169.78,125.59L173.64,126.69L177.5,128.95L181.36,130.38L185.21,138.51L196.79,138.25L200.65,139.44L204.51,138.5L208.37,138.36L212.23,138.9L216.08,143.74L219.94,144.38L223.8,143.28L235.38,142.43L239.24,143.32L243.09,142.87L246.95,145.91L250.81,146.52L254.67,145.26L266.25,144.24L270.1,145.13L273.96,144.33L277.82,147.9L281.68,147.87L285.54,146.98L300.97,145.4L304.83,146.29L308.69,149.5L312.55,149.05L316.41,147.52L320.27,147.61L324.13,146.65L327.98,147.64L335.7,146L339.56,149.34L343.42,149.32L347.28,148.33L354.99,148.62L358.85,146.68L366.57,146.81L370.43,148.02L378.15,149.76L382.01,149.62L389.72,151.19L393.58,151.08L405.16,153.04L409.02,152.44L412.87,153.36L416.73,152.85L439.88,154.48L443.74,153.92L451.46,154.74L455.32,154.56L459.18,154.9L463.04,157.41L470.75,159.8L474.61,160.48L482.33,161.72L486.19,163.29L501.62,165.38L505.48,164.47L532.49,167.38L536.35,166.72L540.21,167.26L544.07,166.74L551.79,166.87L555.64,168.82L563.36,168.82L567.22,168L582.65,167.75L586.51,168.83L594.23,168.31L598.09,167.88L601.95,168.3L605.81,167.68L613.52,166.74L617.38,167.9L636.68,166.35L640.53,166.66L648.25,166.11L652.11,166.24L667.54,164.76L671.4,164.41L675.26,164.4L679.12,165.51L682.98,164.93L686.84,165.64L690.7,164.08L694.55,163.93L698.41,163.27L702.27,163.55L706.13,162.53L709.99,162.91L713.85,163.78L717.71,162.38L733.14,162.64L737,161.91\" style=\"vector-effect: none; fill: none; stroke: rgb(99, 110, 250); stroke-opacity: 1; stroke-width: 2px; opacity: 1;\"/></g><g class=\"points\"/><g class=\"text\"/></g></g></g><g class=\"overplot\"/><path class=\"xlines-above crisp\" d=\"M0,0\" style=\"fill: none;\"/><path class=\"ylines-above crisp\" d=\"M0,0\" style=\"fill: none;\"/><g class=\"overlines-above\"/><g class=\"xaxislayer-above\"><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" transform=\"translate(91.58,0) rotate(90,0,327)\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\">4</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(107.01,0) rotate(90,0,327)\">8</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(122.45,0) rotate(90,0,327)\">12</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(137.88,0) rotate(90,0,327)\">16</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(153.31,0) rotate(90,0,327)\">20</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(168.75,0) rotate(90,0,327)\">24</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(184.18,0) rotate(90,0,327)\">28</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(199.62,0) rotate(90,0,327)\">32</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(215.05,0) rotate(90,0,327)\">36</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(230.49,0) rotate(90,0,327)\">40</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(245.92,0) rotate(90,0,327)\">44</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(261.36,0) rotate(90,0,327)\">48</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(276.78999999999996,0) rotate(90,0,327)\">52</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(292.23,0) rotate(90,0,327)\">56</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(307.65999999999997,0) rotate(90,0,327)\">60</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(323.09000000000003,0) rotate(90,0,327)\">64</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(338.53,0) rotate(90,0,327)\">68</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(353.96,0) rotate(90,0,327)\">72</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(369.4,0) rotate(90,0,327)\">76</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(384.83,0) rotate(90,0,327)\">80</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(400.27,0) rotate(90,0,327)\">84</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(415.7,0) rotate(90,0,327)\">88</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(431.14,0) rotate(90,0,327)\">92</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(446.57,0) rotate(90,0,327)\">96</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(462.01,0) rotate(90,0,327)\">100</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(477.44,0) rotate(90,0,327)\">104</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(492.87,0) rotate(90,0,327)\">108</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(508.31,0) rotate(90,0,327)\">112</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(523.74,0) rotate(90,0,327)\">116</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(539.1800000000001,0) rotate(90,0,327)\">120</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(554.61,0) rotate(90,0,327)\">124</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(570.05,0) rotate(90,0,327)\">128</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(585.48,0) rotate(90,0,327)\">132</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(600.92,0) rotate(90,0,327)\">136</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(616.35,0) rotate(90,0,327)\">140</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(631.79,0) rotate(90,0,327)\">144</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(647.22,0) rotate(90,0,327)\">148</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(662.65,0) rotate(90,0,327)\">152</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(678.09,0) rotate(90,0,327)\">156</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(693.52,0) rotate(90,0,327)\">160</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(708.96,0) rotate(90,0,327)\">164</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(724.39,0) rotate(90,0,327)\">168</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(739.83,0) rotate(90,0,327)\">172</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(755.26,0) rotate(90,0,327)\">176</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(770.7,0) rotate(90,0,327)\">180</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(786.13,0) rotate(90,0,327)\">184</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(801.57,0) rotate(90,0,327)\">188</text></g><g class=\"xtick\"><text text-anchor=\"start\" x=\"0\" y=\"333\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(817,0) rotate(90,0,327)\">192</text></g></g><g class=\"yaxislayer-above\"><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" transform=\"translate(0,309)\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\">12x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,269.4)\">43x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,229.8)\">74x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,190.2)\">104x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,150.6)\">135x</text></g><g class=\"ytick\"><text text-anchor=\"end\" x=\"79\" y=\"4.199999999999999\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre; opacity: 1;\" transform=\"translate(0,111)\">166x</text></g></g><g class=\"overaxes-above\"/></g></g><g class=\"polarlayer\"/><g class=\"smithlayer\"/><g class=\"ternarylayer\"/><g class=\"geolayer\"/><g class=\"funnelarealayer\"/><g class=\"pielayer\"/><g class=\"iciclelayer\"/><g class=\"treemaplayer\"/><g class=\"sunburstlayer\"/><g class=\"glimages\"/><defs id=\"topdefs-cbb66b\"><g class=\"clips\"/><clipPath id=\"legendcbb66b\"><rect width=\"356\" height=\"48\" x=\"0\" y=\"0\"/></clipPath></defs><g class=\"layer-above\"><g class=\"imagelayer\"/><g class=\"shapelayer\"/></g><g class=\"infolayer\"><g class=\"legend\" pointer-events=\"all\" transform=\"translate(831.74,100)\"><rect class=\"bg\" shape-rendering=\"crispEdges\" width=\"356\" height=\"48\" x=\"0\" y=\"0\" style=\"stroke: rgb(68, 68, 68); stroke-opacity: 1; fill: rgb(255, 255, 255); fill-opacity: 1; stroke-width: 0px;\"/><g class=\"scrollbox\" transform=\"\" clip-path=\"url(#legendcbb66b)\"><text class=\"legendtitletext\" text-anchor=\"start\" x=\"2\" y=\"18.2\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 14px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre;\">Configuration</text><g class=\"groups\" transform=\"\"><g class=\"traces\" transform=\"translate(0,32.7)\" style=\"opacity: 1;\"><text class=\"legendtext\" text-anchor=\"start\" x=\"40\" y=\"4.680000000000001\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 12px; fill: rgb(42, 63, 95); fill-opacity: 1; white-space: pre;\">Multicore: Persistent mode/shared memory + kernel config</text><g class=\"layers\" style=\"opacity: 1;\"><g class=\"legendfill\"/><g class=\"legendlines\"><path class=\"js-line\" d=\"M5,0h30\" style=\"fill: none; stroke: rgb(99, 110, 250); stroke-opacity: 1; stroke-width: 2px;\"/></g><g class=\"legendsymbols\"><g class=\"legendpoints\"/></g></g><rect class=\"legendtoggle\" x=\"0\" y=\"-9.5\" width=\"350.46875\" height=\"19\" style=\"fill: rgb(0, 0, 0); fill-opacity: 0;\"/></g></g></g><rect class=\"scrollbar\" rx=\"20\" ry=\"3\" width=\"0\" height=\"0\" x=\"0\" y=\"0\" style=\"fill: rgb(128, 139, 164); fill-opacity: 1;\"/></g><g class=\"g-gtitle\"><text class=\"gtitle\" x=\"60\" y=\"50\" text-anchor=\"start\" dy=\"0em\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 17px; fill: rgb(42, 63, 95); opacity: 1; font-weight: normal; white-space: pre;\">Fuzzer performance</text></g><g class=\"g-xtitle\"><text class=\"xtitle\" x=\"448.5\" y=\"371.659375\" text-anchor=\"middle\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 14px; fill: rgb(42, 63, 95); opacity: 1; font-weight: normal; white-space: pre;\">Number of parallel fuzzers</text></g><g class=\"g-ytitle\"><text class=\"ytitle\" transform=\"rotate(-90,28.668750000000003,210)\" x=\"28.668750000000003\" y=\"210\" text-anchor=\"middle\" style=\"font-family: 'Open Sans', verdana, arial, sans-serif; font-size: 14px; fill: rgb(42, 63, 95); opacity: 1; font-weight: normal; white-space: pre;\">Fuzz target executions per second</text></g></g></svg>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "r6a_pivotdf = r6a_graphdf.pivot(index=\"parallel_fuzzers\", columns=\"label\", values=\"execs_per_sec\")\n", + "r6a_fig = r6a_pivotdf.plot(\n", + " title=\"Fuzzer performance\",\n", + " labels={\n", + " \"label\": \"Configuration\",\n", + " \"parallel_fuzzers\": \"Number of parallel fuzzers\",\n", + " \"value\": \"Fuzz target executions per second\"\n", + " }\n", + ")\n", + "\n", + "# Compute tick values and their labels for the primary Y-axis\n", + "tickvals = np.linspace(r6a_graphdf['execs_per_sec'].min(), r6a_graphdf['execs_per_sec'].max(), 6)\n", + "ticktext = [f\"{val:.0f}x\" for val in tickvals / graphdf['execs_per_sec'].min()]\n", + "# Update the primary Y-axis with custom tick labels\n", + "r6a_fig.update_yaxes(tickvals=tickvals, ticktext=ticktext)\n", + "r6a_fig.update_xaxes(tickvals=list(range(0,200+1, 4)))\n", + "r6a_fig.update_layout(width=1200, height=400)\n", + "r6a_fig.show(\"svg\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Line graph analysis\n", + "\n", + "This is a shocking result for a 192 vCPU machine -- our optimal number of parallel fuzzers was 16! Using 32 parallel fuzzers gives less performance than using 8 fuzzers. Using 192 parallel fuzzers (the physical number of threads in this machine) gives the same performance as using 4 fuzzers.\n", + "\n", + "This is clearly a cautionary tale about measuring before simply using the number of hardware threads in your machine. But does this mean that AFL++ is a bad fuzzer, or that AWS tricked us and gave us a 16-thread machine instead of a 192-thread one?\n", + "\n", + "No, probably not -- the most likely cause here (other than a horrible bug) may be that we're already saturating the Linux kernel's ability to service system calls (although we're definitely hitting such a limit way earlier than I expected). A good way to test this theory would be to run more system-call-servicers (read: kernels!) at once on this machine; one way to do that is to use hardware virtualization with KVM. " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py new file mode 100755 index 00000000..fffb4a3a --- /dev/null +++ b/benchmark/benchmark.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# Part of the aflplusplus project, requires Python 3.8+. +# Author: Chris Ball <chris@printf.net>, ported from Marc "van Hauser" Heuse's "benchmark.sh". +import argparse, asyncio, json, multiprocessing, os, platform, re, shutil, sys +from dataclasses import asdict, dataclass +from decimal import Decimal +from enum import Enum, auto +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +blue = lambda text: f"\033[1;94m{text}\033[0m"; gray = lambda text: f"\033[1;90m{text}\033[0m" +green = lambda text: f"\033[0;32m{text}\033[0m"; red = lambda text: f"\033[0;31m{text}\033[0m" +yellow = lambda text: f"\033[0;33m{text}\033[0m" + +class Mode(Enum): + multicore = auto() + singlecore = auto() + +@dataclass +class Target: + source: Path + binary: Path + +@dataclass +class Run: + execs_per_sec: float + execs_total: float + fuzzers_used: int + +@dataclass +class Config: + afl_persistent_config: bool + afl_system_config: bool + afl_version: Optional[str] + comment: str + compiler: str + target_arch: str + +@dataclass +class Hardware: + cpu_fastest_core_mhz: float + cpu_model: str + cpu_threads: int + +@dataclass +class Results: + config: Optional[Config] + hardware: Optional[Hardware] + targets: Dict[str, Dict[str, Optional[Run]]] + +all_modes = [Mode.singlecore, Mode.multicore] +all_targets = [ + Target(source=Path("../utils/persistent_mode/test-instr.c").resolve(), binary=Path("test-instr-persist-shmem")), + Target(source=Path("../test-instr.c").resolve(), binary=Path("test-instr")) +] +modes = [mode.name for mode in all_modes] +targets = [str(target.binary) for target in all_targets] +cpu_count = multiprocessing.cpu_count() +env_vars = { + "AFL_DISABLE_TRIM": "1", "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES": "1", "AFL_FAST_CAL": "1", + "AFL_NO_UI": "1", "AFL_TRY_AFFINITY": "1", "PATH": f'{str(Path("../").resolve())}:{os.environ["PATH"]}', +} + +parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("-b", "--basedir", help="directory to use for temp files", type=str, default="/tmp/aflpp-benchmark") +parser.add_argument("-d", "--debug", help="show verbose debugging output", action="store_true") +parser.add_argument("-r", "--runs", help="how many runs to average results over", type=int, default=3) +parser.add_argument("-f", "--fuzzers", help="how many afl-fuzz workers to use", type=int, default=cpu_count) +parser.add_argument("-m", "--mode", help="pick modes", action="append", default=modes, choices=modes) +parser.add_argument("-c", "--comment", help="add a comment about your setup", type=str, default="") +parser.add_argument("--cpu", help="override the detected CPU model name", type=str, default="") +parser.add_argument("--mhz", help="override the detected CPU MHz", type=str, default="") +parser.add_argument( + "-t", "--target", help="pick targets", action="append", default=["test-instr-persist-shmem"], choices=targets +) +args = parser.parse_args() +# Really unsatisfying argparse behavior: we want a default and to allow multiple choices, but if there's a manual choice +# it should override the default. Seems like we have to remove the default to get that and have correct help text? +if len(args.target) > 1: + args.target = args.target[1:] +if len(args.mode) > 2: + args.mode = args.mode[2:] + +chosen_modes = [mode for mode in all_modes if mode.name in args.mode] +chosen_targets = [target for target in all_targets if str(target.binary) in args.target] +results = Results(config=None, hardware=None, targets={ + str(t.binary): {m.name: None for m in chosen_modes} for t in chosen_targets} +) +debug = lambda text: args.debug and print(blue(text)) + +async def clean_up_tempfiles() -> None: + shutil.rmtree(f"{args.basedir}/in") + for target in chosen_targets: + target.binary.unlink() + for mode in chosen_modes: + shutil.rmtree(f"{args.basedir}/out-{mode.name}-{str(target.binary)}") + +async def check_afl_persistent() -> bool: + with open("/proc/cmdline", "r") as cmdline: + return "mitigations=off" in cmdline.read().strip().split(" ") + +async def check_afl_system() -> bool: + sysctl = next((s for s in ["sysctl", "/sbin/sysctl"] if shutil.which(s)), None) + if sysctl: + (returncode, stdout, _) = await run_command([sysctl, "kernel.randomize_va_space"]) + return returncode == 0 and stdout.decode().rstrip().split(" = ")[1] == "0" + return False + +async def prep_env() -> None: + Path(f"{args.basedir}/in").mkdir(exist_ok=True, parents=True) + with open(f"{args.basedir}/in/in.txt", "wb") as seed: + seed.write(b"\x00" * 10240) + +async def compile_target(source: Path, binary: Path) -> None: + print(f" [*] Compiling the {binary} fuzzing harness for the benchmark to use.") + (returncode, stdout, stderr) = await run_command( + [str(Path("../afl-clang-lto").resolve()), "-o", str(Path(binary.resolve())), str(Path(source).resolve())] + ) + if returncode == 0: + return + print(yellow(f" [*] afl-clang-lto was unable to compile; falling back to afl-cc.")) + (returncode, stdout, stderr) = await run_command( + [str(Path("../afl-cc").resolve()), "-o", str(Path(binary.resolve())), str(Path(source).resolve())] + ) + if returncode != 0: + sys.exit(red(f" [*] Error: afl-cc is unable to compile: {stderr.decode()} {stdout.decode()}")) + +async def run_command(cmd: List[str]) -> Tuple[Optional[int], bytes, bytes]: + debug(f"Launching command: {cmd} with env {env_vars}") + p = await asyncio.create_subprocess_exec( + *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=env_vars + ) + stdout, stderr = await p.communicate() + debug(f"Output: {stdout.decode()} {stderr.decode()}") + return (p.returncode, stdout, stderr) + +async def check_deps() -> None: + if not (plat := platform.system()) == "Linux": sys.exit(red(f" [*] {plat} is not supported by this script yet.")) + if not os.access(Path("../afl-fuzz").resolve(), os.X_OK) and os.access(Path("../afl-cc").resolve(), os.X_OK) and ( + os.path.exists(Path("../SanitizerCoveragePCGUARD.so").resolve())): + sys.exit(red(" [*] Compile AFL++: we need afl-fuzz, afl-clang-fast and SanitizerCoveragePCGUARD.so built.")) + + (returncode, stdout, stderr) = await run_command([str(Path("../afl-cc").resolve()), "-v"]) + if returncode != 0: + sys.exit(red(f" [*] Error: afl-cc -v returned: {stderr.decode()} {stdout.decode()}")) + compiler = "" + target_arch = "" + for line in stderr.decode().split("\n"): + if "clang version" in line: + compiler = line + elif m := re.match(r"^Target: (.*)", line): + target_arch = m.group(1) + + # Pick some sample settings from afl-{persistent,system}-config to try to see whether they were run. + afl_pc = await check_afl_persistent() + afl_sc = await check_afl_system() + if not afl_pc: + print(yellow(f" [*] afl-persistent-config did not run; run it to improve performance (and decrease security).")) + if not afl_sc: + print(yellow(f" [*] afl-system-config did not run; run it to improve performance (and decrease security).")) + results.config = Config(afl_persistent_config=afl_pc, afl_system_config=afl_sc, afl_version="", + comment=args.comment, compiler=compiler, target_arch=target_arch) + +async def colon_values(filename: str, searchKey: str) -> List[str]: + """Return a colon-separated value given a key in a file, e.g. 'cpu MHz : 4976.109')""" + with open(filename, "r") as fh: + kv_pairs = (line.split(": ", 1) for line in fh if ": " in line) + v_list = [v.rstrip() for k, v in kv_pairs if k.rstrip() == searchKey] + return v_list + +async def describe_afl_config() -> str: + if results.config is None: + return "unknown" + elif results.config.afl_persistent_config and results.config.afl_system_config: + return "both" + elif results.config.afl_persistent_config: + return "persistent" + elif results.config.afl_system_config: + return "system" + else: + return "none" + +async def save_benchmark_results() -> None: + """Append a single row to the benchmark results in JSON Lines format (which is simple to write and diff).""" + with open("benchmark-results.jsonl", "a") as jsonfile: + json.dump(asdict(results), jsonfile, sort_keys=True) + jsonfile.write("\n") + print(blue(f" [*] Results have been written to the {jsonfile.name} file.")) + with open("COMPARISON.md", "r+") as comparisonfile: + described_config = await describe_afl_config() + aflconfig = described_config.ljust(12) + if results.hardware is None: + return + cpu_model = results.hardware.cpu_model.ljust(51) + if cpu_model in comparisonfile.read(): + print(blue(f" [*] Results have not been written to the COMPARISON.md file; this CPU is already present.")) + return + cpu_mhz = str(round(results.hardware.cpu_fastest_core_mhz)).ljust(5) + if not "test-instr-persist-shmem" in results.targets or \ + not "multicore" in results.targets["test-instr-persist-shmem"] or \ + not "singlecore" in results.targets["test-instr-persist-shmem"] or \ + results.targets["test-instr-persist-shmem"]["singlecore"] is None or \ + results.targets["test-instr-persist-shmem"]["multicore"] is None: + return + single = str(round(results.targets["test-instr-persist-shmem"]["singlecore"].execs_per_sec)).ljust(10) + multi = str(round(results.targets["test-instr-persist-shmem"]["multicore"].execs_per_sec)).ljust(9) + cores = str(args.fuzzers).ljust(7) + comparisonfile.write(f"|{cpu_model} | {cpu_mhz} | {cores} | {single} | {multi} | {aflconfig} |\n") + print(blue(f" [*] Results have been written to the COMPARISON.md file.")) + with open("COMPARISON.md", "r") as comparisonfile: + print(comparisonfile.read()) + + +async def main() -> None: + try: + await clean_up_tempfiles() + except FileNotFoundError: + pass + await check_deps() + if args.mhz: + cpu_mhz = float(args.mhz) + else: + cpu_mhz_str = await colon_values("/proc/cpuinfo", "cpu MHz") + if len(cpu_mhz_str) == 0: + cpu_mhz_str.append("0") + cpu_mhz = max([float(c) for c in cpu_mhz_str]) # use the fastest CPU MHz for now + if args.cpu: + cpu_model = [args.cpu] + else: + cpu_model = await colon_values("/proc/cpuinfo", "model name") or [""] + results.hardware = Hardware(cpu_fastest_core_mhz=cpu_mhz, cpu_model=cpu_model[0], cpu_threads=cpu_count) + await prep_env() + print(f" [*] Ready, starting benchmark...") + for target in chosen_targets: + await compile_target(target.source, target.binary) + binary = str(target.binary) + for mode in chosen_modes: + if mode == Mode.multicore: + print(blue(f" [*] Using {args.fuzzers} fuzzers for multicore fuzzing "), end="") + print(blue("(use --fuzzers to override)." if args.fuzzers == cpu_count else f"(the default is {cpu_count})")) + execs_per_sec, execs_total = ([] for _ in range(2)) + for run_idx in range(0, args.runs): + print(gray(f" [*] {mode.name} {binary} run {run_idx+1} of {args.runs}, execs/s: "), end="", flush=True) + fuzzers = range(0, args.fuzzers if mode == Mode.multicore else 1) + outdir = f"{args.basedir}/out-{mode.name}-{binary}" + cmds = [] + for fuzzer_idx, afl in enumerate(fuzzers): + name = ["-o", outdir, "-M" if fuzzer_idx == 0 else "-S", str(afl)] + cmds.append(["afl-fuzz", "-i", f"{args.basedir}/in"] + name + ["-s", "123", "-V10", "-D", f"./{binary}"]) + # Prepare the afl-fuzz tasks, and then block while waiting for them to finish. + fuzztasks = [run_command(cmds[cpu]) for cpu in fuzzers] + await asyncio.gather(*fuzztasks) + afl_versions = await colon_values(f"{outdir}/0/fuzzer_stats", "afl_version") + if results.config: + results.config.afl_version = afl_versions[0] + # Our score is the sum of all execs_per_sec entries in fuzzer_stats files for the run. + sectasks = [colon_values(f"{outdir}/{afl}/fuzzer_stats", "execs_per_sec") for afl in fuzzers] + all_execs_per_sec = await asyncio.gather(*sectasks) + execs = sum([Decimal(count[0]) for count in all_execs_per_sec]) + print(green(execs)) + execs_per_sec.append(execs) + # Also gather execs_total and total_run_time for this run. + exectasks = [colon_values(f"{outdir}/{afl}/fuzzer_stats", "execs_done") for afl in fuzzers] + all_execs_total = await asyncio.gather(*exectasks) + execs_total.append(sum([Decimal(count[0]) for count in all_execs_total])) + + # (Using float() because Decimal() is not JSON-serializable.) + avg_afl_execs_per_sec = round(Decimal(sum(execs_per_sec) / len(execs_per_sec)), 2) + afl_execs_total = int(sum([Decimal(execs) for execs in execs_total])) + run = Run(execs_per_sec=float(avg_afl_execs_per_sec), execs_total=afl_execs_total, fuzzers_used=len(fuzzers)) + results.targets[binary][mode.name] = run + print(f" [*] Average execs/sec for this test across all runs was: {green(avg_afl_execs_per_sec)}") + if (((max(execs_per_sec) - min(execs_per_sec)) / avg_afl_execs_per_sec) * 100) > 15: + print(yellow(" [*] The difference between your slowest and fastest runs was >15%, maybe try again?")) + + await clean_up_tempfiles() + await save_benchmark_results() + +if __name__ == "__main__": + asyncio.run(main()) + diff --git a/custom_mutators/README.md b/custom_mutators/README.md index a5a572c0..2d1220b3 100644 --- a/custom_mutators/README.md +++ b/custom_mutators/README.md @@ -70,14 +70,17 @@ requires cmake (among other things): ### libprotobuf Mutators -There are two WIP protobuf projects, that require work to be working though: +There are three WIP protobuf projects, that require work to be working though: + +ASN.1 example: +[https://github.com/airbus-seclab/AFLplusplus-blogpost/tree/main/src/mutator](https://github.com/airbus-seclab/AFLplusplus-blogpost/tree/main/src/mutator) transforms protobuf raw: -https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator +[https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator) has a transform function you need to fill for your protobuf format, however needs to be ported to the updated AFL++ custom mutator API (not much work): -https://github.com/thebabush/afl-libprotobuf-mutator +[https://github.com/thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator) same as above but is for current AFL++: -https://github.com/P1umer/AFLplusplus-protobuf-mutator +[https://github.com/P1umer/AFLplusplus-protobuf-mutator](https://github.com/P1umer/AFLplusplus-protobuf-mutator) \ No newline at end of file diff --git a/custom_mutators/aflpp/Makefile b/custom_mutators/aflpp/Makefile new file mode 100644 index 00000000..8efdf3e4 --- /dev/null +++ b/custom_mutators/aflpp/Makefile @@ -0,0 +1,10 @@ + +CFLAGS = -O3 -funroll-loops -fPIC -Wl,-Bsymbolic + +all: aflpp-mutator.so + +aflpp-mutator.so: aflpp.c + $(CC) $(CFLAGS) -I../../include -I. -shared -o aflpp-mutator.so aflpp.c ../../src/afl-performance.c + +clean: + rm -f *.o *~ *.so core diff --git a/custom_mutators/aflpp/README.md b/custom_mutators/aflpp/README.md new file mode 100644 index 00000000..04d605c1 --- /dev/null +++ b/custom_mutators/aflpp/README.md @@ -0,0 +1,8 @@ +# custum mutator: AFL++ + +this is the AFL++ havoc mutator as a custom mutator module for AFL++. + +just type `make` to build + +```AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/aflpp/aflpp-mutator.so afl-fuzz ...``` + diff --git a/custom_mutators/aflpp/aflpp.c b/custom_mutators/aflpp/aflpp.c new file mode 100644 index 00000000..0b236f76 --- /dev/null +++ b/custom_mutators/aflpp/aflpp.c @@ -0,0 +1,90 @@ +#include "afl-fuzz.h" +#include "afl-mutations.h" + +typedef struct my_mutator { + + afl_state_t *afl; + u8 *buf; + u32 buf_size; + +} my_mutator_t; + +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { + + (void)seed; + + my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + if ((data->buf = malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init alloc"); + return NULL; + + } else { + + data->buf_size = MAX_FILE; + + } + + data->afl = afl; + + return data; + +} + +/* here we run the AFL++ mutator, which is the best! */ + +size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, + u8 **out_buf, uint8_t *add_buf, size_t add_buf_size, + size_t max_size) { + + if (max_size > data->buf_size) { + + u8 *ptr = realloc(data->buf, max_size); + + if (ptr) { + + return 0; + + } else { + + data->buf = ptr; + data->buf_size = max_size; + + } + + } + + u32 havoc_steps = 1 + rand_below(data->afl, 16); + + /* set everything up, costly ... :( */ + memcpy(data->buf, buf, buf_size); + + /* the mutation */ + u32 out_buf_len = afl_mutate(data->afl, data->buf, buf_size, havoc_steps, + false, true, add_buf, add_buf_size, max_size); + + /* return size of mutated data */ + *out_buf = data->buf; + return out_buf_len; + +} + +/** + * Deinitialize everything + * + * @param data The data ptr from afl_custom_init + */ +void afl_custom_deinit(my_mutator_t *data) { + + free(data->buf); + free(data); + +} + diff --git a/custom_mutators/aflpp/standalone/Makefile b/custom_mutators/aflpp/standalone/Makefile new file mode 100644 index 00000000..f1e99445 --- /dev/null +++ b/custom_mutators/aflpp/standalone/Makefile @@ -0,0 +1,10 @@ + +CFLAGS = -O3 -funroll-loops -fPIC + +all: aflpp-standalone + +aflpp-standalone: aflpp-standalone.c + $(CC) $(CFLAGS) -I../../../include -I. -o aflpp-standalone aflpp-standalone.c ../../../src/afl-performance.c + +clean: + rm -f *.o *~ aflpp-standalone core diff --git a/custom_mutators/aflpp/standalone/README.md b/custom_mutators/aflpp/standalone/README.md new file mode 100644 index 00000000..a1ffb5f9 --- /dev/null +++ b/custom_mutators/aflpp/standalone/README.md @@ -0,0 +1,10 @@ +# AFL++ standalone mutator + +this is the AFL++ havoc mutator as a standalone mutator + +just type `make` to build. + +``` +aflpp-standalone inputfile outputfile [splicefile] +``` + diff --git a/custom_mutators/aflpp/standalone/aflpp-standalone.c b/custom_mutators/aflpp/standalone/aflpp-standalone.c new file mode 100644 index 00000000..3a2cbc2f --- /dev/null +++ b/custom_mutators/aflpp/standalone/aflpp-standalone.c @@ -0,0 +1,163 @@ +#include "afl-fuzz.h" +#include "afl-mutations.h" + +typedef struct my_mutator { + + afl_state_t *afl; + u8 *buf; + u32 buf_size; + +} my_mutator_t; + +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { + + (void)seed; + + my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + if ((data->buf = malloc(1024*1024)) == NULL) { + + perror("afl_custom_init alloc"); + return NULL; + + } else { + + data->buf_size = 1024*1024; + + } + + /* fake AFL++ state */ + data->afl = calloc(1, sizeof(afl_state_t)); + data->afl->queue_cycle = 1; + data->afl->fsrv.dev_urandom_fd = open("/dev/urandom", O_RDONLY); + if (data->afl->fsrv.dev_urandom_fd < 0) { PFATAL("Unable to open /dev/urandom"); } + rand_set_seed(data->afl, getpid()); + + return data; + +} + +/* here we run the AFL++ mutator, which is the best! */ + +size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, + u8 **out_buf, uint8_t *add_buf, size_t add_buf_size, + size_t max_size) { + + if (max_size > data->buf_size) { + + u8 *ptr = realloc(data->buf, max_size); + + if (ptr) { + + return 0; + + } else { + + data->buf = ptr; + data->buf_size = max_size; + + } + + } + + u32 havoc_steps = 1 + rand_below(data->afl, 16); + + /* set everything up, costly ... :( */ + memcpy(data->buf, buf, buf_size); + + /* the mutation */ + u32 out_buf_len = afl_mutate(data->afl, data->buf, buf_size, havoc_steps, + false, true, add_buf, add_buf_size, max_size); + + /* return size of mutated data */ + *out_buf = data->buf; + return out_buf_len; + +} + +int main(int argc, char *argv[]) { + + if (argc > 1 && strncmp(argv[1], "-h", 2) == 0) { + printf("Syntax: %s [-v] [inputfile [outputfile [splicefile]]]\n\n", argv[0]); + printf("Reads a testcase from stdin when no input file (or '-') is specified,\n"); + printf("mutates according to AFL++'s mutation engine, and write to stdout when '-' or\n"); + printf("no output filename is given. As an optional third parameter you can give a file\n"); + printf("for splicing. Maximum input and output length is 1MB.\n"); + printf("The -v verbose option prints debug output to stderr.\n"); + return 0; + } + + FILE *in = stdin, *out = stdout, *splice = NULL; + unsigned char *inbuf = malloc(1024 * 1024), *outbuf, *splicebuf = NULL; + int verbose = 0, splicelen = 0; + + if (argc > 1 && strcmp(argv[1], "-v") == 0) { + verbose = 1; + argc--; + argv++; + fprintf(stderr, "Verbose active\n"); + } + + my_mutator_t *data = afl_custom_init(NULL, 0); + + if (argc > 1 && strcmp(argv[1], "-") != 0) { + if ((in = fopen(argv[1], "r")) == NULL) { + perror(argv[1]); + return -1; + } + if (verbose) fprintf(stderr, "Input: %s\n", argv[1]); + } + + size_t inlen = fread(inbuf, 1, 1024*1024, in); + + if (!inlen) { + fprintf(stderr, "Error: empty file %s\n", argv[1] ? argv[1] : "stdin"); + return -1; + } + + if (argc > 2 && strcmp(argv[2], "-") != 0) { + if ((out = fopen(argv[2], "w")) == NULL) { + perror(argv[2]); + return -1; + } + if (verbose) fprintf(stderr, "Output: %s\n", argv[2]); + } + + if (argc > 3) { + if ((splice = fopen(argv[3], "r")) == NULL) { + perror(argv[3]); + return -1; + } + if (verbose) fprintf(stderr, "Splice: %s\n", argv[3]); + splicebuf = malloc(1024*1024); + size_t splicelen = fread(splicebuf, 1, 1024*1024, splice); + if (!splicelen) { + fprintf(stderr, "Error: empty file %s\n", argv[3]); + return -1; + } + if (verbose) fprintf(stderr, "Mutation splice length: %zu\n", splicelen); + } + + if (verbose) fprintf(stderr, "Mutation input length: %zu\n", inlen); + unsigned int outlen = afl_custom_fuzz(data, inbuf, inlen, &outbuf, splicebuf, splicelen, 1024*1024); + + if (outlen == 0 || !outbuf) { + fprintf(stderr, "Error: no mutation data returned.\n"); + return -1; + } + + if (verbose) fprintf(stderr, "Mutation output length: %u\n", outlen); + + if (fwrite(outbuf, 1, outlen, out) != outlen) { + fprintf(stderr, "Warning: incomplete write.\n"); + return -1; + } + + return 0; +} diff --git a/custom_mutators/aflpp_tritondse/README.md b/custom_mutators/aflpp_tritondse/README.md index 8a5dd02b..033655d2 100644 --- a/custom_mutators/aflpp_tritondse/README.md +++ b/custom_mutators/aflpp_tritondse/README.md @@ -10,8 +10,13 @@ ../../afl-cc -o ../../test-instr ../../test-instr.c mkdir -p in echo aaaa > in/in -TRITON_DSE_TARGET=../../test-instr AFL_CUSTOM_MUTATOR_ONLY=1 AFL_SYNC_TIME=1 AFL_PYTHON_MODULE=aflpp_tritondse PYTHONPATH=. ../../afl-fuzz -i in -o out -- ../../test-instr +AFL_DISABLE_TRIM=1 AFL_CUSTOM_MUTATOR_ONLY=1 AFL_SYNC_TIME=1 AFL_PYTHON_MODULE=aflpp_tritondse PYTHONPATH=. ../../afl-fuzz -i in -o out -- ../../test-instr ``` Note that this custom mutator works differently, new finds are synced -after 10-60 seconds to the fuzzing instance. +after 10-60 seconds to the fuzzing instance. This is necessary because only +C/C++ custom mutators have access to the internal AFL++ state. + +Note that you should run first with `AFL_DEBUG` for 5-10 minutes and see if +all important libraries and syscalls are hooked (look at `WARNING` and `CRITICAL` +output during the run, best use with `AFL_NO_UI=1`) diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py index e0219f0b..58739696 100644 --- a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -22,14 +22,17 @@ config = None dse = None cycle = 0 count = 0 +finding = 0 hashes = set() format = SeedFormat.RAW def pre_exec_hook(se: SymbolicExecutor, state: ProcessState): global count global hashes + global finding if se.seed.hash not in hashes: hashes.add(se.seed.hash) + finding = 1 filename = out_path + "/id:" + f"{count:06}" + "," + se.seed.hash if not os.path.exists(filename): if is_debug: @@ -47,6 +50,59 @@ def pre_exec_hook(se: SymbolicExecutor, state: ProcessState): # file.write(se.seed.content) +#def rtn_open(se: SymbolicExecutor, pstate: ProcessState, pc): +# """ +# The open behavior. +# """ +# logging.debug('open hooked') +# +# # Get arguments +# arg0 = pstate.get_argument_value(0) # const char *pathname +# flags = pstate.get_argument_value(1) # int flags +# mode = pstate.get_argument_value(2) # int mode +# arg0s = pstate.memory.read_string(arg0) +# +# # Concretize the whole path name +# pstate.concretize_memory_bytes(arg0, len(arg0s)+1) # Concretize the whole string + \0 +# +# # We use flags as concrete value +# pstate.concretize_argument(1) +# +# # Use the flags to open the file in the write mode. +# mode = "" +# if (flags & 0xFF) == 0x00: # O_RDONLY +# mode = "r" +# elif (flags & 0xFF) == 0x01: # O_WRONLY +# mode = "w" +# elif (flags & 0xFF) == 0x02: # O_RDWR +# mode = "r+" +# +# if (flags & 0x0100): # O_CREAT +# mode += "x" +# if (flags & 0x0200): # O_APPEND +# mode = "a" # replace completely value +# +# if se.seed.is_file_defined(arg0s) and "r" in mode: # input file and opened in reading +# logging.info(f"opening an input file: {arg0s}") +# # Program is opening an input +# data = se.seed.get_file_input(arg0s) +# filedesc = pstate.create_file_descriptor(arg0s, io.BytesIO(data)) +# fd = filedesc.id +# else: +# # Try to open it as a regular file +# try: +# fd = open(arg0s, mode) # use the mode here +# filedesc = pstate.create_file_descriptor(arg0s, fd) +# fd = filedesc.id +# except Exception as e: +# logging.debug(f"Failed to open {arg0s} {e}") +# fd = pstate.minus_one +# +# pstate.write_register("rax", fd) # write the return value +# pstate.cpu.program_counter = pstate.pop_stack_value() # pop the return value +# se.skip_instruction() # skip the current instruction so that the engine go straight fetching the next instruction + + def init(seed): global config global dse @@ -64,6 +120,10 @@ def init(seed): is_debug = True except KeyError: pass + if is_debug: + logging.basicConfig(level=logging.WARNING) + else: + logging.basicConfig(level=logging.CRITICAL) try: foo = os.environ['AFL_CUSTOM_INFO_OUT'] out_path = foo + '/../tritondse/queue' @@ -104,7 +164,7 @@ def init(seed): format = SeedFormat.COMPOSITE # Now set up TritonDSE config = Config(coverage_strategy = CoverageStrategy.PATH, - debug = is_debug, + # debug = is_debug, pipe_stdout = is_debug, pipe_stderr = is_debug, execution_timeout = 1, @@ -115,10 +175,16 @@ def init(seed): dse = SymbolicExplorator(config, prog) # Add callbacks. dse.callback_manager.register_pre_execution_callback(pre_exec_hook) + #dse.callback_manager.register_function_callback("open", rtn_open) -#def fuzz(buf, add_buf, max_size): -# return b"" +def fuzz(buf, add_buf, max_size): + global finding + finding = 1 + while finding == 1: + finding = 0 + dse.step() + return b"" def queue_new_entry(filename_new_queue, filename_orig_queue): @@ -141,8 +207,14 @@ def queue_new_entry(filename_new_queue, filename_orig_queue): dse.add_input_seed(seed) # Start exploration! #dse.step() - dse.explore() + #dse.explore() pass + +# we simulate just doing one single fuzz in the custom mutator +def fuzz_count(buf): + return 1 + + def splice_optout(): pass diff --git a/custom_mutators/examples/README.md b/custom_mutators/examples/README.md index 655f7a5e..112db243 100644 --- a/custom_mutators/examples/README.md +++ b/custom_mutators/examples/README.md @@ -33,3 +33,6 @@ like surgical_havoc_mutate() that allow to perform a randomly chosen mutation from a subset of the havoc mutations. If you do so, you have to specify -I /path/to/AFLplusplus/include when compiling. + +elf_header_mutator.c - example ELF header mutator based on + [LibGolf](https://github.com/xcellerator/libgolf/) diff --git a/custom_mutators/examples/custom_post_run.c b/custom_mutators/examples/custom_post_run.c new file mode 100644 index 00000000..828216ea --- /dev/null +++ b/custom_mutators/examples/custom_post_run.c @@ -0,0 +1,53 @@ +// +// This is an example on how to use afl_custom_post_run +// It executes custom code each time after AFL++ executes the target +// +// cc -O3 -fPIC -shared -g -o custom_post_run.so -I../../include custom_post_run.c +// cd ../.. +// afl-cc -o test-instr test-instr.c +// AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/examples/custom_post_run.so \ +// afl-fuzz -i in -o out -- ./test-instr -f /tmp/foo +// + + +#include "afl-fuzz.h" + +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <stdio.h> + +typedef struct my_mutator { + + afl_state_t *afl; + +} my_mutator_t; + +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { + + my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + data->afl = afl; + + return data; + +} + +void afl_custom_post_run(my_mutator_t *data) { + + printf("hello from afl_custom_post_run\n"); + return; +} + + +void afl_custom_deinit(my_mutator_t *data) { + + free(data); + +} \ No newline at end of file diff --git a/custom_mutators/examples/elf_header_mutator.c b/custom_mutators/examples/elf_header_mutator.c new file mode 100644 index 00000000..b985257a --- /dev/null +++ b/custom_mutators/examples/elf_header_mutator.c @@ -0,0 +1,679 @@ +/* + AFL++ Custom Mutator for ELF Headers + Written by @echel0n <melih.sahin@protonmail.com> + based on libgolf.h by @xcellerator + $ gcc -O3 -fPIC -shared -o elf_mutator.so -I ~/AFLplusplus/include/ + */ +#include "afl-fuzz.h" +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <limits.h> +#include <linux/elf.h> + +/* EI_ABIVERSION isn't used anymore and elf.h defines EI_PAD to be 0x09 */ +#define EI_ABIVERSION 0x08 +#define EI_PAD 0x09 +/* Define the Architecture and ISA constants to match those in <linux/elf.h> */ +#define X86_64 EM_X86_64 +#define ARM32 EM_ARM +#define AARCH64 EM_AARCH64 +#define uchar unsigned char +#define DATA_SIZE 0x100 + +/* + * The ELF and Program headers are different sizes depending on 32- and 64-bit + * architectures + * taken from libgolf.h + */ +#define EHDR_T(x) Elf##x##_Ehdr +#define PHDR_T(x) Elf##x##_Phdr +#define EHDR(x) ehdr##x +#define PHDR(x) phdr##x +#define GET_EHDR(x) (&(elf_ptr->EHDR(x))); +#define GET_PHDR(x) (&(elf_ptr->PHDR(x))); +#define REF_EHDR(b, x) ((Elf##b##_Ehdr *)ehdr)->x +#define REF_PHDR(b, x) ((Elf##b##_Phdr *)phdr)->x +int ehdr_size; +int phdr_size; +/* + * This struct holds the bytes that will be executed, and the size. + */ +typedef struct text_segment { + + size_t text_size; + unsigned char *text_segment; + +} TextSegment; + +// example shellcode that exits +// taken from libgolf.h +unsigned char buf[] = {0xb0, 0x3c, 0x31, 0xff, 0x0f, 0x05}; + +/* + * This is the raw ELF file + * - EHDR(xx) is the ELF header + * - PHDR(xx) is the program header + * - text is the text segment + * - filename is the name of the golf'd binary + * - isa is the target architecture (X86_64, ARM32, AARCH64) + * taken from libgolf.h + */ +typedef struct rawbinary_t { + + EHDR_T(32) EHDR(32); + PHDR_T(32) PHDR(32); + EHDR_T(64) EHDR(64); + PHDR_T(64) PHDR(64); + TextSegment text; + char *filename; + int isa; + +} RawBinary; + +/* + * Copy an E_IDENT array into the corresponding fields in the ELF header + * Called by populate_ehdr() + * taken from libgolf.h + */ +int populate_e_ident(RawBinary *elf_ptr, unsigned char e_ident[]) { + + int i; + /* Depending on whether the target ISA is 32- or 64-bit, set e_ident */ + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + for (i = 0; i < EI_NIDENT; i++) + elf_ptr->EHDR(64).e_ident[i] = e_ident[i]; + break; + case ARM32: + for (i = 0; i < EI_NIDENT; i++) + elf_ptr->EHDR(32).e_ident[i] = e_ident[i]; + break; + default: + exit(1); + + } + + return 0; + +} + +/* + * Copy bytes from buf[] array into text_segment in ELF struct + * taken from libgolf.h + */ +int copy_text_segment(RawBinary *elf_ptr, unsigned char buf[], int text_size) { + + int i; + + /* Set size of text segment and allocate the buffer */ + elf_ptr->text.text_size = text_size; + elf_ptr->text.text_segment = + malloc(elf_ptr->text.text_size * sizeof(unsigned char)); + + /* Copy the bytes into the text segment buffer */ + for (i = 0; i < elf_ptr->text.text_size; i++) { + + elf_ptr->text.text_segment[i] = buf[i]; + + } + +} + +/* + * Populate the ELF Header with sane values + * Returns a pointer to an EHDR struct + * taken from libgolf.h + */ +void *populate_ehdr(RawBinary *elf_ptr) { + + /* + * Set ehdr_size and phdr_size. Determined by whether target ISA is 32- or + * 64-bit. + */ + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + ehdr_size = sizeof(EHDR_T(64)); + phdr_size = sizeof(PHDR_T(64)); + break; + case ARM32: + ehdr_size = sizeof(EHDR_T(32)); + phdr_size = sizeof(PHDR_T(32)); + break; + default: + exit(1); + + }; + + /* Start with the E_IDENT area at the top of the file */ + unsigned char e_ident[EI_NIDENT] = {0}; + + /* Magic Bytes */ + e_ident[EI_MAG0] = 0x7F; + e_ident[EI_MAG1] = 0x45; // E + e_ident[EI_MAG2] = 0x4C; // L + e_ident[EI_MAG3] = 0x46; // F + + /* + * EI_CLASS denotes the architecture: + * ELFCLASS32: 0x01 + * ELFCLASS64: 0x02 + */ + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + e_ident[EI_CLASS] = ELFCLASS64; + break; + case ARM32: + e_ident[EI_CLASS] = ELFCLASS32; + break; + default: + exit(1); + + } + + /* + * EI_DATA denotes the endianness: + * ELFDATA2LSB: 0x01 + * ELFDATA2MSB: 0x02 + */ + e_ident[EI_DATA] = ELFDATA2LSB; + + /* EI_VERSION is always 0x01 */ + e_ident[EI_VERSION] = EV_CURRENT; + + /* + * EI_OSABI defines the target OS. Ignored by most modern ELF parsers. + */ + e_ident[EI_OSABI] = ELFOSABI_NONE; + + /* EI_ABIVERSION was for sub-classification. Un-defined since Linux 2.6 */ + e_ident[EI_ABIVERSION] = 0x00; + + /* EI_PAD is currently unused */ + e_ident[EI_PAD] = 0x00; + + /* Copy the E_IDENT section to the ELF struct */ + populate_e_ident(elf_ptr, e_ident); + + /* + * The remainder of the ELF header following E_IDENT follows. + * + * ehdr is a pointer to either an Elf32_Edhr, or Elf64_Ehdr struct. + */ + void *ehdr = NULL; + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + ehdr = (&(elf_ptr->EHDR(64))); + break; + case ARM32: + ehdr = (&(elf_ptr->EHDR(32))); + break; + default: + exit(1); + + } + + /* + * Depending on whether the ISA is 32- or 64-bit determines the size of + * many of the fields in the ELF Header. This switch case deals with it. + */ + switch (elf_ptr->isa) { + + // 64-Bit ISAs + case X86_64: + case AARCH64: + /* + * e_type specifies what kind of ELF file this is: + * ET_NONE: 0x00 // Unknown Type + * ET_REL: 0x01 // Relocatable + * ET_EXEC: 0x02 // Executable File + * ET_DYN: 0x03 // Shared Object + * ET_CORE: 0x04 // Core Dump + */ + REF_EHDR(64, e_type) = ET_EXEC; // 0x0002 + + /* e_machine specifies the target ISA */ + REF_EHDR(64, e_machine) = elf_ptr->isa; + + /* e_version is always set of 0x01 for the original ELF spec */ + REF_EHDR(64, e_version) = EV_CURRENT; // 0x00000001 + + /* + * e_entry is the memory address of the entry point + * Set by set_entry_point() after p_vaddr is set in the phdr + */ + REF_EHDR(64, e_entry) = 0x0; + + /* + * e_phoff points to the start of the program header, which + * immediately follows the ELF header + */ + REF_EHDR(64, e_phoff) = ehdr_size; + + /* e_shoff points to the start of the section header table */ + REF_EHDR(64, e_shoff) = 0x00; + + /* e_flags is architecture dependent */ + REF_EHDR(64, e_flags) = 0x0; + + /* e_ehsize contains the size of the ELF header */ + REF_EHDR(64, e_ehsize) = ehdr_size; + + /* e_phentsize is the size of the program header */ + REF_EHDR(64, e_phentsize) = phdr_size; + + /* + * e_phnum contains the number of entries in the program header + * e_phnum * e_phentsize = size of program header table + */ + REF_EHDR(64, e_phnum) = 0x1; + + /* e_shentsize contains the size of a section header entry */ + REF_EHDR(64, e_shentsize) = 0x0; + + /* + * e_shnum contains the number of entries in the section header + * e_shnum * e_shentsize = size of section header table + */ + REF_EHDR(64, e_shnum) = 0x0; + + /* + * e_shstrndx contains the index of the section header table that + * contains the section names + */ + REF_EHDR(64, e_shstrndx) = 0x0; + + break; + // 32-Bit ISAs + case ARM32: + /* + * e_type specifies what kind of ELF file this is: + * ET_NONE: 0x00 // Unknown Type + * ET_REL: 0x01 // Relocatable + * ET_EXEC: 0x02 // Executable File + * ET_DYN: 0x03 // Shared Object + * ET_CORE: 0x04 // Core Dump + */ + REF_EHDR(32, e_type) = ET_EXEC; // 0x0002 + + /* e_machine specifies the target ISA */ + REF_EHDR(32, e_machine) = elf_ptr->isa; + + /* e_version is always set of 0x01 for the original ELF spec */ + REF_EHDR(32, e_version) = EV_CURRENT; // 0x00000001 + + /* + * e_entry is the memory address of the entry point + * Set by set_entry_point() after p_vaddr is set in the phdr + */ + REF_EHDR(32, e_entry) = 0x0; + + /* + * e_phoff points to the start of the program header, which + * immediately follows the ELF header + */ + REF_EHDR(32, e_phoff) = ehdr_size; + + /* e_shoff points to the start of the section header table */ + REF_EHDR(32, e_shoff) = 0x0i; + + /* e_flags is architecture dependent */ + REF_EHDR(32, e_flags) = 0x0; + + /* e_ehsize contains the size of the ELF header */ + REF_EHDR(32, e_ehsize) = ehdr_size; + + /* e_phentsize is the size of the program header */ + REF_EHDR(32, e_phentsize) = phdr_size; + + /* + * e_phnum contains the number of entries in the program header + * e_phnum * e_phentsize = size of program header table + */ + REF_EHDR(32, e_phnum) = 0x1; + + /* e_shentsize contains the size of a section header entry */ + REF_EHDR(32, e_shentsize) = 0x0; + + /* + * e_shnum contains the number of entries in the section header + * e_shnum * e_shentsize = size of section header table + */ + REF_EHDR(32, e_shnum) = 0x0; + + /* + * e_shstrndx contains the index of the section header table that + * contains the section names + */ + REF_EHDR(32, e_shnum) = 0x0; + + break; + + } + + return ehdr; + +} + +/* + * Populate the program headers with sane values + * Returns a pointer to a PHDR struct + * taken from libgolf.h + */ +void *populate_phdr(RawBinary *elf_ptr) { + + /* + * All offsets are relative to the start of the program header (0x40) + * + * phdr is a pointer to either an Elf32_Phdr, or Elf64_Phdr struct. + */ + void *phdr = NULL; + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + phdr = (&(elf_ptr->PHDR(64))); + break; + case ARM32: + phdr = (&(elf_ptr->PHDR(32))); + break; + default: + exit(1); + + } + + /* + * Depending on whether the ISA is 32- or 64-bit determines the size of + * many of the fields in the Progra Header. This switch case deals with it. + */ + switch (elf_ptr->isa) { + + // 64-Bit ISAs + case X86_64: + case AARCH64: + /* + * p_type identifies what type of segment this is + * PT_NULL: 0x0 // Unused + * PT_LOAD: 0x1 // Loadable Segment + * PT_DYNAMIC: 0x2 // Dynamic Linker Information + * PT_INTERP: 0x3 // Interpreter Information + * PT_NOTE: 0x4 // Auxiliary Information + * PT_SHLIB: 0x5 // Reserved + * PT_PHDR: 0x6 // Segment with Program Header + * PT_TLS: 0x7 // Thread Local Storage + */ + REF_PHDR(64, p_type) = PT_LOAD; // 0x1 + + /* + * p_flags defines permissions for this section + * PF_R: 0x4 // Read + * PF_W: 0x2 // Write + * PF_X: 0x1 // Execute + */ + REF_PHDR(64, p_flags) = PF_R | PF_X; // 0x5 + + /* + * p_offset is the offset in the file image (relative to the start + * of the program header) for this segment. + */ + REF_PHDR(64, p_offset) = 0x0; + + /* + * p_vaddr is the virtual address where this segment should be loaded + * p_paddr is for the physical address (unused by System V) + */ + REF_PHDR(64, p_vaddr) = 0x400000; + REF_PHDR(64, p_paddr) = 0x400000; + + /* + * p_filesz is the size of the segment in the file image + * p_memsz is the size of the segment in memory + * + * Note: p_filesz doesn't have to equal p_memsz + */ + REF_PHDR(64, p_filesz) = elf_ptr->text.text_size; + REF_PHDR(64, p_memsz) = elf_ptr->text.text_size; + + break; + // 32-Bit ISAs + case ARM32: + /* + * p_type identifies what type of segment this is + * PT_NULL: 0x0 // Unused + * PT_LOAD: 0x1 // Loadable Segment + * PT_DYNAMIC: 0x2 // Dynamic Linker Information + * PT_INTERP: 0x3 // Interpreter Information + * PT_NOTE: 0x4 // Auxiliary Information + * PT_SHLIB: 0x5 // Reserved + * PT_PHDR: 0x6 // Segment with Program Header + * PT_TLS: 0x7 // Thread Local Storage + */ + REF_PHDR(32, p_type) = PT_LOAD; // 0x1 + + /* + * p_flags defines permissions for this section + * PF_R: 0x4 // Read + * PF_W: 0x2 // Write + * PF_X: 0x1 // Execute + */ + REF_PHDR(32, p_flags) = PF_R | PF_X; // 0x5 + + /* + * p_offset is the offset in the file image (relative to the start + * of the program header) for this segment. + */ + REF_PHDR(32, p_offset) = 0x0; + + /* + * p_vaddr is the virtual address where this segment should be loaded + * p_paddr is for the physical address (unused by System V) + */ + REF_PHDR(32, p_vaddr) = 0x10000; + REF_PHDR(32, p_paddr) = 0x10000; + + /* + * p_filesz is the size of the segment in the file image + * p_memsz is the size of the segment in memory + * + * Note: p_filesz doesn't have to equal p_memsz + */ + REF_PHDR(32, p_filesz) = elf_ptr->text.text_size; + REF_PHDR(32, p_memsz) = elf_ptr->text.text_size; + + break; + default: + exit(1); + + } + + /* + * p_align is the memory alignment + * + * Note: p_vaddr = p_offset % p_align + */ + switch (elf_ptr->isa) { + + case X86_64: + REF_PHDR(64, p_align) = 0x400000; + break; + case ARM32: + REF_PHDR(32, p_align) = 0x10000; + break; + case AARCH64: + REF_PHDR(64, p_align) = 0x400000; + break; + + } + + return phdr; + +} + +/* + * e_entry depends on p_vaddr, so has to be set after populate_ehdr() + * and populate_phdr() have been called. + * taken from libgolf.h + */ +int set_entry_point(RawBinary *elf_ptr) { + + /* + * Once the whole ELF file is copied into memory, control is handed to + * e_entry. Relative to the process's virtual memory address, the .text + * segment will be located immediately after the ELF and program header. + * + * ehdr and phdr are pointers to the ELF and Program headers respectively. + * The switch case casts and assigns them to the correct fields of the ELF + * struct, then sets ehdr->e_entry. + */ + void *ehdr, *phdr; + + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + ehdr = GET_EHDR(64); + phdr = GET_PHDR(64); + REF_EHDR(64, e_entry) = REF_PHDR(64, p_vaddr) + ehdr_size + phdr_size; + break; + case ARM32: + ehdr = GET_EHDR(32); + phdr = GET_PHDR(32); + REF_EHDR(32, e_entry) = REF_PHDR(32, p_vaddr) + ehdr_size + phdr_size; + break; + default: + exit(1); + + } + + return 0; + +} + +typedef struct my_mutator { + + afl_state_t *afl; + size_t trim_size_current; + int trimmming_steps; + int cur_step; + u8 *mutated_out, *post_process_buf, *trim_buf; + +} my_mutator_t; + +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { + + srand(seed); // needed also by surgical_havoc_mutate() + my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + if ((data->mutated_out = (u8 *)malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init malloc"); + return NULL; + + } + + if ((data->post_process_buf = (u8 *)malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init malloc"); + return NULL; + + } + + if ((data->trim_buf = (u8 *)malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init malloc"); + return NULL; + + } + + data->afl = afl; + return data; + +} + +size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *in_buf, size_t buf_size, + u8 **out_buf, uint8_t *add_buf, + size_t add_buf_size, // add_buf can be NULL + size_t max_size) { + + RawBinary elf_obj; + RawBinary *elf = &elf_obj; + elf->isa = 62; + Elf64_Ehdr *ehdr; + Elf64_Phdr *phdr; + copy_text_segment(elf, buf, sizeof(buf)); + ehdr = populate_ehdr(elf); + phdr = populate_phdr(elf); + set_entry_point(elf); + + size_t mutated_size = ehdr_size + phdr_size + elf->text.text_size; + int pos = 0; + // example fields + ehdr->e_ident[EI_CLASS] = (uint8_t *)(in_buf + pos++); + ehdr->e_ident[EI_DATA] = (uint8_t *)(in_buf + pos++); + ehdr->e_ident[EI_VERSION] = (uint8_t *)(in_buf + pos++); + ehdr->e_ident[EI_OSABI] = (uint8_t *)(in_buf + pos++); + for (int i = 0x8; i < 0x10; ++i) { + + (ehdr->e_ident)[i] = (uint8_t *)(in_buf + pos++); + + } + + ehdr->e_version = (uint32_t *)(in_buf + pos); + pos += 4; + // sections headers + ehdr->e_shoff = (uint64_t *)(in_buf + pos); + pos += 8; + ehdr->e_shentsize = (uint16_t *)(in_buf + pos); + pos += 2; + ehdr->e_shnum = (uint16_t *)(in_buf + pos); + pos += 2; + ehdr->e_shstrndx = (uint16_t *)(in_buf + pos); + pos += 2; + ehdr->e_flags = (uint32_t *)(in_buf + pos); + pos += 4; + // physical addr + phdr->p_paddr = (uint64_t *)(in_buf + pos); + pos += 8; + phdr->p_align = (uint64_t *)(in_buf + pos); + pos += 8; + + /* mimic GEN_ELF() + * Write: + * - ELF Header + * - Program Header + * - Text Segment + */ + memcpy(data->mutated_out, ehdr, ehdr_size); + memcpy(data->mutated_out + ehdr_size, phdr, phdr_size); + memcpy(data->mutated_out + ehdr_size + phdr_size, elf->text.text_segment, + elf->text.text_size); + + *out_buf = data->mutated_out; + return mutated_size; + +} + +void afl_custom_deinit(my_mutator_t *data) { + + free(data->post_process_buf); + free(data->mutated_out); + free(data->trim_buf); + free(data); + +} + diff --git a/custom_mutators/examples/example.py b/custom_mutators/examples/example.py index 3a6d22e4..830f302f 100644 --- a/custom_mutators/examples/example.py +++ b/custom_mutators/examples/example.py @@ -133,6 +133,11 @@ def fuzz(buf, add_buf, max_size): # @return: The buffer containing the test case after # ''' # return buf +# def post_run(): +# ''' +# Called after each time the execution of the target program by AFL++ +# ''' +# pass # # def havoc_mutation(buf, max_size): # ''' diff --git a/custom_mutators/grammar_mutator/GRAMMAR_VERSION b/custom_mutators/grammar_mutator/GRAMMAR_VERSION index 2568c6a5..3a019448 100644 --- a/custom_mutators/grammar_mutator/GRAMMAR_VERSION +++ b/custom_mutators/grammar_mutator/GRAMMAR_VERSION @@ -1 +1 @@ -ff4e5a2 +5ed4f8d diff --git a/custom_mutators/grammar_mutator/grammar_mutator b/custom_mutators/grammar_mutator/grammar_mutator -Subproject ff4e5a265daf5d88c4a636fb6a2c22b1d733db0 +Subproject 5ed4f8d6e6524df9670af6b411b13031833d67d diff --git a/custom_mutators/symcc/README.md b/custom_mutators/symcc/README.md index 364a348e..a6839a37 100644 --- a/custom_mutators/symcc/README.md +++ b/custom_mutators/symcc/README.md @@ -5,6 +5,8 @@ This uses the symcc to find new paths into the target. Note that this is a just a proof of concept example! It is better to use the fuzzing helpers of symcc, symqemu, Fuzzolic, etc. rather than this. +Also the symqemu custom mutator is better than this. + To use this custom mutator follow the steps in the symcc repository [https://github.com/eurecom-s3/symcc/](https://github.com/eurecom-s3/symcc/) on how to build symcc and how to instrument a target binary (the same target diff --git a/custom_mutators/symqemu/Makefile b/custom_mutators/symqemu/Makefile new file mode 100644 index 00000000..958aec19 --- /dev/null +++ b/custom_mutators/symqemu/Makefile @@ -0,0 +1,14 @@ + +ifdef DEBUG + CFLAGS += -DDEBUG +endif + +all: symqemu-mutator.so + +CFLAGS += -O3 -funroll-loops + +symqemu-mutator.so: symqemu.c + $(CC) -g $(CFLAGS) $(CPPFLAGS) -g -I../../include -shared -fPIC -o symqemu-mutator.so symqemu.c + +clean: + rm -f symqemu-mutator.so *.o *~ core diff --git a/custom_mutators/symqemu/README.md b/custom_mutators/symqemu/README.md new file mode 100644 index 00000000..c3071afc --- /dev/null +++ b/custom_mutators/symqemu/README.md @@ -0,0 +1,19 @@ +# custum mutator: symqemu + +This uses the symcc to find new paths into the target. + +## How to build and use + +To use this custom mutator follow the steps in the symqemu repository +[https://github.com/eurecom-s3/symqemu/](https://github.com/eurecom-s3/symqemu/) +on how to build symqemu-x86_x64 and put it in your `PATH`. + +Just type `make` to build this custom mutator. + +```AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/symqemu/symqemu-mutator.so AFL_DISABLE_TRIM=1 afl-fuzz ...``` + +## Options + +`SYMQEMU_ALL=1` - use concolic solving on **all** queue items, not only interesting/favorite ones. + +`SYMQEMU_LATE=1` - use concolic solving only after there have been no finds for 5 minutes. diff --git a/custom_mutators/symqemu/symqemu.c b/custom_mutators/symqemu/symqemu.c new file mode 100644 index 00000000..73a1640a --- /dev/null +++ b/custom_mutators/symqemu/symqemu.c @@ -0,0 +1,424 @@ +#define _GNU_SOURCE +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <ctype.h> +#include "config.h" +#include "debug.h" +#include "afl-fuzz.h" +#include "common.h" + +afl_state_t *afl_struct; +static u32 debug = 0; +static u32 found_items = 0; + +#define SYMQEMU_LOCATION "symqemu" + +#define DBG(x...) \ + if (debug) { fprintf(stderr, x); } + +typedef struct my_mutator { + + afl_state_t *afl; + u32 all; + u32 late; + u8 *mutator_buf; + u8 *out_dir; + u8 *target; + u8 *symqemu; + u8 *input_file; + u32 counter; + u32 seed; + u32 argc; + u8 **argv; + +} my_mutator_t; + +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { + + if (getenv("AFL_DEBUG")) debug = 1; + + my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + char *path = getenv("PATH"); + char *exec_name = "symqemu-x86_64"; + char *token = strtok(path, ":"); + char exec_path[4096]; + + while (token != NULL && data->symqemu == NULL) { + + snprintf(exec_path, sizeof(exec_path), "%s/%s", token, exec_name); + if (access(exec_path, X_OK) == 0) { + + data->symqemu = (u8 *)strdup(exec_path); + break; + + } + + token = strtok(NULL, ":"); + + } + + if (!data->symqemu) FATAL("symqemu binary %s not found", exec_name); + DBG("Found %s\n", data->symqemu); + + if (getenv("AFL_CUSTOM_MUTATOR_ONLY")) { + + WARNF( + "the symqemu module is not very effective with " + "AFL_CUSTOM_MUTATOR_ONLY."); + + } + + if ((data->mutator_buf = malloc(MAX_FILE)) == NULL) { + + free(data); + perror("mutator_buf alloc"); + return NULL; + + } + + data->target = getenv("AFL_CUSTOM_INFO_PROGRAM"); + + u8 *path_tmp = getenv("AFL_CUSTOM_INFO_OUT"); + u32 len = strlen(path_tmp) + 32; + u8 *symqemu_path = malloc(len); + data->out_dir = malloc(len); + snprintf(symqemu_path, len, "%s/%s", path_tmp, SYMQEMU_LOCATION); + snprintf(data->out_dir, len, "%s/out", symqemu_path, path_tmp); + + (void)mkdir(symqemu_path, 0755); + (void)mkdir(data->out_dir, 0755); + + setenv("SYMCC_OUTPUT_DIR", data->out_dir, 1); + + data->input_file = getenv("AFL_CUSTOM_INFO_PROGRAM_INPUT"); + + u8 *tmp = NULL; + if ((tmp = getenv("AFL_CUSTOM_INFO_PROGRAM_ARGV")) && *tmp) { + + int argc = 0, index = 2; + for (u32 i = 0; i < strlen(tmp); ++i) + if (isspace(tmp[i])) ++argc; + + data->argv = (u8 **)malloc((argc + 4) * sizeof(u8 **)); + u8 *p = strdup(tmp); + + do { + + data->argv[index] = p; + while (*p && !isspace(*p)) + ++p; + if (*p) { + + *p++ = 0; + while (isspace(*p)) + ++p; + + } + + if (strcmp(data->argv[index], "@@") == 0) { + + if (!data->input_file) { + + u32 ilen = strlen(symqemu_path) + 32; + data->input_file = malloc(ilen); + snprintf(data->input_file, ilen, "%s/.input", symqemu_path); + + } + + data->argv[index] = data->input_file; + + } + + DBG("%d: %s\n", index, data->argv[index]); + index++; + + } while (*p); + + data->argv[index] = NULL; + data->argc = index; + + } else { + + data->argv = (u8 **)malloc(8 * sizeof(u8 **)); + data->argc = 2; + data->argv[2] = NULL; + + } + + data->argv[0] = data->symqemu; + data->argv[1] = data->target; + data->afl = afl; + data->seed = seed; + afl_struct = afl; + + if (getenv("SYMQEMU_ALL")) { data->all = 1; } + if (getenv("SYMQEMU_LATE")) { data->late = 1; } + if (data->input_file) { setenv("SYMCC_INPUT_FILE", data->input_file, 1); } + + DBG("out_dir=%s, target=%s, input_file=%s, argc=%u\n", data->out_dir, + data->target, + data->input_file ? (char *)data->input_file : (char *)"<stdin>", + data->argc); + + if (debug) { + + fprintf(stderr, "["); + for (u32 i = 0; i <= data->argc; ++i) + fprintf(stderr, " \"%s\"", + data->argv[i] ? (char *)data->argv[i] : "<NULL>"); + fprintf(stderr, " ]\n"); + + } + + return data; + +} + +/* No need to receive a splicing item */ +void afl_custom_splice_optout(void *data) { + + (void)(data); + +} + +/* Get unix time in milliseconds */ + +inline u64 get_cur_time(void) { + + struct timeval tv; + struct timezone tz; + + gettimeofday(&tv, &tz); + + return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000); + +} + +u32 afl_custom_fuzz_count(my_mutator_t *data, const u8 *buf, size_t buf_size) { + + if (likely((!afl_struct->queue_cur->favored && !data->all) || + afl_struct->queue_cur->was_fuzzed)) { + + return 0; + + } + + if (likely(data->late)) { + + if (unlikely(get_cur_time() - afl_struct->last_find_time <= + 10 * 60 * 1000)) { + + return 0; + + } + + } + + int pipefd[2]; + struct stat st; + + if (afl_struct->afl_env.afl_no_ui) { + + ACTF("Sending to symqemu: %s", afl_struct->queue_cur->fname); + + } + + if (!(stat(afl_struct->queue_cur->fname, &st) == 0 && S_ISREG(st.st_mode) && + st.st_size)) { + + PFATAL("Couldn't find enqueued file: %s", afl_struct->queue_cur->fname); + + } + + if (afl_struct->fsrv.use_stdin) { + + if (pipe(pipefd) == -1) { + + PFATAL( + "Couldn't create a pipe for interacting with symqemu child process"); + + } + + } + + if (data->input_file) { + + int fd = open(data->input_file, O_WRONLY | O_CREAT | O_TRUNC, 0644); + ssize_t s = write(fd, buf, buf_size); + close(fd); + DBG("wrote %zd/%zd to %s\n", s, buf_size, data->input_file); + + } + + int pid = fork(); + + if (pid == -1) return 0; + + if (likely(pid)) { + + if (!data->input_file || afl_struct->fsrv.use_stdin) { + + close(pipefd[0]); + + if (fcntl(pipefd[1], F_GETPIPE_SZ)) { + + fcntl(pipefd[1], F_SETPIPE_SZ, MAX_FILE); + + } + + ck_write(pipefd[1], buf, buf_size, data->input_file); + + close(pipefd[1]); + + } + + pid = waitpid(pid, NULL, 0); + DBG("symqemu finished executing!\n"); + + } else /* (pid == 0) */ { // child + + if (afl_struct->fsrv.use_stdin) { + + close(pipefd[1]); + dup2(pipefd[0], 0); + + } + + DBG("exec=%s\n", data->target); + if (!debug) { + + close(1); + close(2); + dup2(afl_struct->fsrv.dev_null_fd, 1); + dup2(afl_struct->fsrv.dev_null_fd, 2); + + } + + execvp((char *)data->argv[0], (char **)data->argv); + fprintf(stderr, "Executing: ["); + for (u32 i = 0; i <= data->argc; ++i) + fprintf(stderr, " \"%s\"", + data->argv[i] ? (char *)data->argv[i] : "<NULL>"); + fprintf(stderr, " ]\n"); + FATAL("Failed to execute %s %s\n", data->argv[0], data->argv[1]); + exit(-1); + + } + + /* back in mother process */ + + struct dirent **nl; + s32 i, items = scandir(data->out_dir, &nl, NULL, NULL); + found_items = 0; + char source_name[4096]; + + if (items > 0) { + + for (i = 0; i < (u32)items; ++i) { + + // symqemu output files start with a digit + if (!isdigit(nl[i]->d_name[0])) continue; + + struct stat st; + snprintf(source_name, sizeof(source_name), "%s/%s", data->out_dir, + nl[i]->d_name); + DBG("file=%s\n", source_name); + + if (stat(source_name, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { + + ++found_items; + + } + + free(nl[i]); + + } + + free(nl); + + } + + DBG("Done, found %u items!\n", found_items); + + return found_items; + +} + +size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, + u8 **out_buf, u8 *add_buf, size_t add_buf_size, + size_t max_size) { + + struct dirent **nl; + s32 done = 0, i, items = scandir(data->out_dir, &nl, NULL, NULL); + char source_name[4096]; + + if (items > 0) { + + for (i = 0; i < (u32)items; ++i) { + + // symqemu output files start with a digit + if (!isdigit(nl[i]->d_name[0])) continue; + + struct stat st; + snprintf(source_name, sizeof(source_name), "%s/%s", data->out_dir, + nl[i]->d_name); + DBG("file=%s\n", source_name); + + if (stat(source_name, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { + + int fd = open(source_name, O_RDONLY); + if (fd < 0) { goto got_an_issue; } + + ssize_t r = read(fd, data->mutator_buf, MAX_FILE); + close(fd); + + DBG("fn=%s, fd=%d, size=%ld\n", source_name, fd, r); + + if (r < 1) { goto got_an_issue; } + + done = 1; + --found_items; + unlink(source_name); + + *out_buf = data->mutator_buf; + return (u32)r; + + } + + free(nl[i]); + + } + + free(nl); + + } + +got_an_issue: + *out_buf = NULL; + return 0; + +} + +/** + * Deinitialize everything + * + * @param data The data ptr from afl_custom_init + */ +void afl_custom_deinit(my_mutator_t *data) { + + free(data->mutator_buf); + free(data); + +} + diff --git a/docs/Changelog.md b/docs/Changelog.md index 3602af50..6736e42b 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -3,7 +3,167 @@ This is the list of all noteworthy changes made in every public release of the tool. See README.md for the general instruction manual. -### Version ++4.07a (dev) +### Version ++4.21a (dev) + * afl-fuzz + - added AFL_DISABLE_REDUNDANT for huge queues + - fix AFL_PERSISTENT_RECORD + - run custom_post_process after standard trimming + - prevent filenames in the queue that have spaces + - minor fix for FAST schedules + - more frequent stats update when syncing (todo: check performance impact) + - now timing of calibration, trimming and syncing is measured seperately, + thanks to @eqv! + - -V timing is now accurately the fuzz time (without syncing), before + long calibration times and syncing could result in now fuzzing being + made when the time was already run out until then, thanks to @eqv! + * afl-cc: + - re-enable i386 support that was accidently disabled + - fixes for LTO and outdated afl-gcc mode for i386 + - fix COMPCOV split compare for old LLVMs + - disable xml/curl/g_ string transform functions because we do not check + for null pointers ... TODO + - ensure shared memory variables are visible in weird build setups + * afl-cmin + - work with input files that have a space + * afl-showmap + - minor fix to collect coverage -C (thanks to @bet4it) + * enhanced the ASAN configuration + + +### Version ++4.20c (release) + ! A new forkserver communication model is now introduced. afl-fuzz is + backward compatible to old compiled targets if they are not built + for CMPLOG/Redqueen, but new compiled targets will not work with + old afl-fuzz versions! + ! Recompile all targets that are instrumented for CMPLOG/Redqueen! + - AFL++ now supports up to 4 billion coverage edges, up from 6 million. + - New compile option: `make PERFORMANCE=1` - this will enable special + CPU dependent optimizations that make everything more performant - but + the binaries will likely won't work on different platforms. Also + enables a faster hasher if the CPU requirements are met. + - The persistent record feature (see config.h) was expanded to also + support replay, thanks to @quarta-qti ! + - afl-fuzz: + - the new deterministic fuzzing feature is now activated by default, + deactivate with -z. Parameters -d and -D are ignored. + - small improvements to CMPLOG/redqueen + - workround for a bug with MOpt -L when used with -M - in the future + we will either remove or rewrite MOpt. + - fix for `-t xxx+` feature + - -e extension option now saves the queue items, crashes, etc. with the + extension too + - fixes for trimmming, correct -V time and reading stats on resume by eqv + thanks a lot! + - afl-cc: + - added collision free caller instrumentation to LTO mode. activate with + `AFL_LLVM_LTO_CALLER=1`. You can set a max depth to go through single + block functions with `AFL_LLVM_LTO_CALLER_DEPTH` (default 0) + - fixes for COMPCOV/LAF and most other modules + - fix for GCC_PLUGIN cmplog that broke on std::strings + - afl-whatsup: + - now also displays current average speed + - small bugfixes + - Fixes for aflpp custom mutator and standalone tool + - Minor edits to afl-persistent-config + - Prevent temporary files being left behind on aborted afl-whatsup + - More CPU benchmarks added to benchmark/ + + +### Version ++4.10c (release) + - afl-fuzz: + - default power schedule is now EXPLORE, due a fix in fast schedules + explore is slightly better now. + - fixed minor issues in the mutation engine, thanks to @futhewo for + reporting! + - better deterministic fuzzing is now available, benchmarks have shown + to improve fuzzing. Enable with -D. Thanks to @kdsjZh for the PR! + - afl-cc: + - large rewrite by @SonicStark which fixes a few corner cases, thanks! + - LTO mode now requires llvm 12+ + - workaround for ASAN with gcc_plugin mode + - instrumentation: + - LLVM 18 support, thanks to @devnexen! + - Injection (SQL, LDAP, XSS) fuzzing feature now available, see + `instrumentation/README.injections.md` how to activate/use/expand. + - compcov/LAF-intel: + - floating point splitting bug fix by @hexcoder + - due a bug in LLVM 17 integer splitting is disabled there! + - when splitting floats was selected, integers were always split as well, + fixed to require AFL_LLVM_LAF_SPLIT_COMPARES or _ALL as it should + - dynamic instrumentation filtering for LLVM NATIVE, thanks @Mozilla! + see utils/dynamic_covfilter/README.md + - qemu_mode: + - plugins are now activated by default and a new module is included that + produces drcov compatible traces for lighthouse/lightkeeper/... + thanks to @JRomainG to submitting! + - updated Nyx checkout (fixes a bug) and some QOL + - updated the custom grammar mutator + - document afl-cmin does not work on macOS (but afl-cmin.bash does) + +### Version ++4.09c (release) + - afl-fuzz: + - fixed the new mutation implementation for two bugs + - added `AFL_FINAL_SYNC` which forces a final fuzzer sync (also for `-F`) + before terminating. + - added AFL_IGNORE_SEED_PROBLEMS to skip over seeds that time out instead + of exiting with an error message + - allow -S/-M naming up to 50 characters (from 24) + - CMPLOG: + - added scale support (-l S) + - skip unhelpful insertions (u8) + - added --version and --help command line parameters + - fixed endless loop when reading malformed dictionaries + - new custom mutator function: post_run - thanks to yangzao! + - afl-whatsup: + - detect instanced that are starting up and show them as such as not dead + - now also shows coverage reached + - option -m shows only very relevant stats + - option -n will not use color in the output + - instrumentation: + - fix for a few string compare transform functions for LAF + - we are instrumenting __cxx internal functions again. this might break + a few targets, please report if so. + - frida_mode: + - fixes support for large map offsets + - support for AFL_FUZZER_LOOPCOUNT for afl.rs and LLVMFuzzerTestOneInput + - afl-cmin/afl-cmin.bash: prevent unneeded file errors + - added new tool afl-addseeds that adds new seeds to a running campaign + - added benchmark/benchmark.py if you want to see how good your fuzzing + speed is in comparison to other setups. + +### Version ++4.08c (release) + - afl-fuzz: + - new mutation engine: mutations that favor discovery more paths are + prefered until no new finds for 10 minutes then switching to mutations + that favor triggering crashes. Modes and switch time can be configured + with `-P`. Also input mode for the target can be defined with `-a` to + be `text` or `binary` (defaults to `generic`) + - new custom mutator that has the new afl++ engine (so it can easily + incorporated into new custom mutators), and also comes with a standalone + command line tool! See custom_mutators/aflpp/standalone/ + - display the state of the fuzzing run in the UI :-) + - fix timeout setting if '+' is used or a session is restarted + - -l X option to enable base64 transformation solving + - allow to disable CMPLOG with '-c -' (e.g. afl.rs enforces '-c 0' on + every instance which is counterproductive). + - afl-cmin/afl-cmin.bash: + - fixed a bug inherited from vanilla AFL where a coverage of + map[123] = 11 would be the same as map[1123] = 1 + - warn on crashing inputs + - adjust threads if less inputs than threads specified + - afl-cc: + - fixed an off-by-one instrumentation of iselect, hurting coverage a bit. + Thanks to @amykweon for spotting and fixing! + - @toka fixed a bug in laf-intel signed integer comparison splitting, + thanks a lot!! + - more LLVM compatability + - frida_mode: + - support for long form instrumentation on x86_x64 and arm64 + - renamed utils/get_symbol_addr.sh to utils/frida_get_symbol_addr.sh + - qemu_mode: + - added qemu_mode/utils/qemu_get_symbol_addr.sh + +### Version ++4.07c (release) - afl-fuzz: - reverse reading the seeds only on restarts (increases performance) - new env `AFL_POST_PROCESS_KEEP_ORIGINAL` to keep the orignal @@ -15,15 +175,21 @@ - new env `AFL_LLVM_LTO_SKIPINIT` to support the AFL++ based WASM (https://github.com/fgsect/WAFL) project - error and print help if afl-clan-lto is used with lto=thin + - rewrote our PCGUARD pass to be compatible with LLVM 15+ shenanigans, + requires LLVM 13+ now instead of 10.0.1+ + - fallback to native LLVM PCGUARD if our PCGUARD is unavailable + - fixed a crash in GCC CMPLOG - afl-showmap: - added custom mutator post_process and send support - add `-I filelist` option, an alternative to `-i in_dir` - afl-cmin + afl-cmin.bash: - `-T threads` parallel task support, can be a huge speedup! - qemu_mode: - - Persistent mode +QASAN support for ppc32 tragets by @worksbutnottested + - Persistent mode + QASAN support for ppc32 targets by @worksbutnottested - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! - + - two new custom mutators are now available: + - TritonDSE in custom_mutators/aflpp_tritondse + - SymQEMU in custom_mutators/symqemu ### Version ++4.06c (release) - afl-fuzz: diff --git a/docs/FAQ.md b/docs/FAQ.md index 8178db46..242a379b 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -29,8 +29,8 @@ If you find an interesting or important question missing, submit it via which then implemented their own research and features, making it now by far the most flexible and feature rich guided fuzzer available as open source. And in independent fuzzing benchmarks it is one of the best fuzzers available, - e.g., [Fuzzbench - Report](https://www.fuzzbench.com/reports/2020-08-03/index.html). + e.g., + [Fuzzbench Report](https://www.fuzzbench.com/reports/2020-08-03/index.html). </p></details> <details> @@ -103,6 +103,42 @@ If you find an interesting or important question missing, submit it via to itself, this too would be an edge. </p></details> +<details> + <summary id="should-you-ever-stop-afl-fuzz-minimize-the-corpus-and-restart">Should you ever stop afl-fuzz, minimize the corpus and restart?</summary><p> + + To stop afl-fuzz, minimize it's corpus and restart you would usually do: + + ``` + Control-C # to terminate afl-fuzz + $ afl-cmin -T nproc -i out/default/queue -o minimized_queue -- ./target + $ AFL_FAST_CAL=1 AFL_CMPLOG_ONLY_NEW=1 afl-fuzz -i minimized_queue -o out2 [other options] -- ./target + ``` + + If this improves fuzzing or not is debated and no consensus has been reached + or in-depth analysis been performed. + + On the pro side: + * The queue/corpus is reduced (up to 20%) by removing intermediate paths + that are maybe not needed anymore. + + On the con side: + * Fuzzing time is lost for the time the fuzzing is stopped, minimized and + restarted. + + The the big question: + * Does a minimized queue/corpus improve finding new coverage or does it + hinder it? + + The AFL++ team's own limited analysis seem to to show that keeping + intermediate paths help to find more coverage, at least for afl-fuzz. + + For honggfuzz in comparison it is a good idea to restart it from time to + time if you have other fuzzers (e.g: AFL++) running in parallel to sync + the finds of other fuzzers to honggfuzz as it has no syncing feature like + AFL++ or libfuzzer. + +</p></details> + ## Targets <details> @@ -279,3 +315,54 @@ If you find an interesting or important question missing, submit it via Solution: just do an `export AFL_MAP_SIZE=(the value in the warning)`. </p></details> + +<details> + <summary id="linker-errors">Linker errors.</summary><p> + + If you compile C++ harnesses and see `undefined reference` errors for + variables named `__afl_...`, e.g.: + + ``` + /usr/bin/ld: /tmp/test-d3085f.o: in function `foo::test()': + test.cpp:(.text._ZN3fooL4testEv[_ZN3fooL4testEv]+0x35): undefined reference to `foo::__afl_connected' + clang: error: linker command failed with exit code 1 (use -v to see invocation) + ``` + + Then you use AFL++ macros like `__AFL_LOOP` within a namespace and this + will not work. + + Solution: Move that harness portion to the global namespace, e.g. before: + ``` + #include <cstdio> + namespace foo { + static void test() { + while(__AFL_LOOP(1000)) { + foo::function(); + } + } + } + + int main(int argc, char** argv) { + foo::test(); + return 0; + } + ``` + after: + ``` + #include <cstdio> + static void mytest() { + while(__AFL_LOOP(1000)) { + foo::function(); + } + } + namespace foo { + static void test() { + mytest(); + } + } + int main(int argc, char** argv) { + foo::test(); + return 0; + } + ``` +</p></details> diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 637e8658..3089aab2 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -3,12 +3,11 @@ ## Linux on x86 An easy way to install AFL++ with everything compiled is available via docker: -You can use the [Dockerfile](../Dockerfile) (which has gcc-10 and clang-12 - -hence afl-clang-lto is available) or just pull directly from the Docker Hub -(for x86_64 and arm64): +You can use the [Dockerfile](../Dockerfile) or just pull directly from the +Docker Hub (for x86_64 and arm64): ```shell -docker pull aflplusplus/aflplusplus: +docker pull aflplusplus/aflplusplus:latest docker run -ti -v /location/of/your/target:/src aflplusplus/aflplusplus ``` @@ -21,14 +20,14 @@ development state of AFL++. If you want to build AFL++ yourself, you have many options. The easiest choice is to build and install everything: -NOTE: depending on your Debian/Ubuntu/Kali/... release, replace `-12` with -whatever llvm version is available. We recommend llvm 12, 13 or 14. +NOTE: depending on your Debian/Ubuntu/Kali/... release, replace `-14` with +whatever llvm version is available. We recommend llvm 13 or newer. ```shell sudo apt-get update sudo apt-get install -y build-essential python3-dev automake cmake git flex bison libglib2.0-dev libpixman-1-dev python3-setuptools cargo libgtk-3-dev -# try to install llvm 12 and install the distro default if that fails -sudo apt-get install -y lld-12 llvm-12 llvm-12-dev clang-12 || sudo apt-get install -y lld llvm llvm-dev clang +# try to install llvm 14 and install the distro default if that fails +sudo apt-get install -y lld-14 llvm-14 llvm-14-dev clang-14 || sudo apt-get install -y lld llvm llvm-dev clang sudo apt-get install -y gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-plugin-dev libstdc++-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-dev sudo apt-get install -y ninja-build # for QEMU mode git clone https://github.com/AFLplusplus/AFLplusplus @@ -68,19 +67,20 @@ These build targets exist: * unit: perform unit tests (based on cmocka) * help: shows these build options -[Unless you are on Mac OS X](https://developer.apple.com/library/archive/qa/qa1118/_index.html), +[Unless you are on macOS](https://developer.apple.com/library/archive/qa/qa1118/_index.html), you can also build statically linked versions of the AFL++ binaries by passing -the `STATIC=1` argument to make: +the `PERFORMANCE=1` argument to make: ```shell -make STATIC=1 +make PERFORMANCE=1 ``` These build options exist: -* STATIC - compile AFL++ static -* CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md) -* ASAN_BUILD - compiles AFL++ with memory sanitizer for debug purposes +* PERFORMANCE - compile with performance options that make the binary not transferable to other systems. Recommended (except on macOS)! +* STATIC - compile AFL++ static (does not work on macOS) +* CODE_COVERAGE - compile the target for code coverage (see [README.llvm.md](../instrumentation/README.llvm.md)) +* ASAN_BUILD - compiles AFL++ with address sanitizer for debug purposes * UBSAN_BUILD - compiles AFL++ tools with undefined behaviour sanitizer for debug purposes * DEBUG - no optimization, -ggdb3, all warnings and -Werror * LLVM_DEBUG - shows llvm deprecation warnings @@ -88,104 +88,82 @@ These build options exist: * INTROSPECTION - compile afl-fuzz with mutation introspection * NO_PYTHON - disable python support * NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for normal fuzzing +* NO_UTF - do not use UTF-8 for line rendering in status screen (fallback to G1 box drawing, of vanilla AFL) * NO_NYX - disable building nyx mode dependencies * NO_CORESIGHT - disable building coresight (arm64 only) * NO_UNICORN_ARM64 - disable building unicorn on arm64 -* AFL_NO_X86 - if compiling on non-intel/amd platforms +* AFL_NO_X86 - if compiling on non-Intel/AMD platforms * LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config (e.g., Debian) e.g.: `make LLVM_CONFIG=llvm-config-14` -## MacOS X on x86 and arm64 (M1) +## macOS on x86_64 and arm64 -MacOS has some gotchas due to the idiosyncrasies of the platform. +macOS has some gotchas due to the idiosyncrasies of the platform. -To build AFL, install llvm (and perhaps gcc) from brew and follow the general -instructions for Linux. If possible, avoid Xcode at all cost. +macOS supports SYSV shared memory used by AFL++'s instrumentation, but the +default settings aren't sufficient. Before even building, increase +them by running the provided script: + +```shell +sudo afl-system-config +``` + +See +[https://www.spy-hill.com/help/apple/SharedMemory.html](https://www.spy-hill.com/help/apple/SharedMemory.html) +for documentation for the shared memory settings and how to make them permanent. + +Next, to build AFL++, install the following packages from brew: ```shell brew install wget git make cmake llvm gdb coreutils ``` -Be sure to setup `PATH` to point to the correct clang binaries and use the -freshly installed clang, clang++, llvm-config, gmake and coreutils, e.g.: +Depending on your macOS system + brew version, brew may be installed in different places. +You can check with `brew info llvm` to know where, then create a variable for it: ```shell -# Depending on your MacOS system + brew version it is either -export PATH="/opt/homebrew/opt/llvm/bin:$PATH" -# or -export PATH="/usr/local/opt/llvm/bin:$PATH" -# you can check with "brew info llvm" +export HOMEBREW_BASE="/opt/homebrew/opt" +``` -export PATH="/usr/local/opt/coreutils/libexec/gnubin:/usr/local/bin:$PATH" +or + +```shell +export HOMEBREW_BASE="/usr/local/opt" +``` + +Set `PATH` to point to the brew clang, clang++, llvm-config, gmake and coreutils. +Also use the brew clang compiler; the Xcode clang compiler must not be used. + +```shell +export PATH="$HOMEBREW_BASE/coreutils/libexec/gnubin:/usr/local/bin:$HOMEBREW_BASE/llvm/bin:$PATH" export CC=clang export CXX=clang++ -gmake -cd frida_mode -gmake -cd .. -sudo gmake install ``` -`afl-gcc` will fail unless you have GCC installed, but that is using outdated -instrumentation anyway. `afl-clang` might fail too depending on your PATH setup. -But you don't want neither, you want `afl-clang-fast` anyway :) Note that -`afl-clang-lto`, `afl-gcc-fast` and `qemu_mode` are not working on MacOS. +Then build following the general Linux instructions. -The crash reporting daemon that comes by default with MacOS X will cause -problems with fuzzing. You need to turn it off: +If everything worked, you should then have `afl-clang-fast` installed, which you can check with: +```shell +which afl-clang-fast ``` -launchctl unload -w /System/Library/LaunchAgents/com.apple.ReportCrash.plist -sudo launchctl unload -w /System/Library/LaunchDaemons/com.apple.ReportCrash.Root.plist -``` -The `fork()` semantics on OS X are a bit unusual compared to other unix systems +Note that `afl-clang-lto`, `afl-gcc-fast` and `qemu_mode` are not working on macOS. + +The crash reporting daemon that comes by default with macOS will cause +problems with fuzzing. You need to turn it off, which you can do with `afl-system-config`. + +The `fork()` semantics on macOS are a bit unusual compared to other unix systems and definitely don't look POSIX-compliant. This means two things: - Fuzzing will be probably slower than on Linux. In fact, some folks report considerable performance gains by running the jobs inside a Linux VM on - MacOS X. + macOS. - Some non-portable, platform-specific code may be incompatible with the AFL++ forkserver. If you run into any problems, set `AFL_NO_FORKSRV=1` in the environment before starting afl-fuzz. -User emulation mode of QEMU does not appear to be supported on MacOS X, so +User emulation mode of QEMU does not appear to be supported on macOS, so black-box instrumentation mode (`-Q`) will not work. However, FRIDA mode (`-O`) -works on both x86 and arm64 MacOS boxes. - -MacOS X supports SYSV shared memory used by AFL's instrumentation, but the -default settings aren't usable with AFL++. The default settings on 10.14 seem to -be: - -```bash -$ ipcs -M -IPC status from <running system> as of XXX -shminfo: - shmmax: 4194304 (max shared memory segment size) - shmmin: 1 (min shared memory segment size) - shmmni: 32 (max number of shared memory identifiers) - shmseg: 8 (max shared memory segments per process) - shmall: 1024 (max amount of shared memory in pages) -``` - -To temporarily change your settings to something minimally usable with AFL++, -run these commands as root: - -```bash -sysctl kern.sysv.shmmax=8388608 -sysctl kern.sysv.shmall=4096 -``` - -If you're running more than one instance of AFL, you likely want to make -`shmall` bigger and increase `shmseg` as well: - -```bash -sysctl kern.sysv.shmmax=8388608 -sysctl kern.sysv.shmseg=48 -sysctl kern.sysv.shmall=98304 -``` - -See -[http://www.spy-hill.com/help/apple/SharedMemory.html](http://www.spy-hill.com/help/apple/SharedMemory.html) -for documentation for these settings and how to make them permanent. +works on both x86 and arm64 macOS boxes. diff --git a/docs/afl-fuzz_approach.md b/docs/afl-fuzz_approach.md index cb173f10..9ea06325 100644 --- a/docs/afl-fuzz_approach.md +++ b/docs/afl-fuzz_approach.md @@ -5,6 +5,10 @@ instrumentation-guided genetic algorithm. It uses a modified form of edge coverage to effortlessly pick up subtle, local-scale changes to program control flow. +Note: If you are interested in a more current up-to-date deep dive how AFL++ +works then we commend this blog post: +[https://blog.ritsec.club/posts/afl-under-hood/](https://blog.ritsec.club/posts/afl-under-hood/) + Simplifying a bit, the overall algorithm can be summed up as: 1) Load user-supplied initial test cases into the queue. @@ -419,8 +423,8 @@ the process. Be sure to consult this file especially if any UI elements are highlighted in red. The fuzzing process will continue until you press Ctrl-C. At a minimum, you want -to allow the fuzzer to complete one queue cycle, which may take anywhere from a -couple of hours to a week or so. +to allow the fuzzer to at least one queue cycle without any new finds, which may +take anywhere from a couple of hours to a week or so. There are three subdirectories created within the output directory and updated in real-time: diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index 3f7e9e6e..b7a7032f 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -73,7 +73,7 @@ def init(seed): def fuzz_count(buf): return cnt -def splice_optout() +def splice_optout(): pass def fuzz(buf, add_buf, max_size): @@ -125,8 +125,9 @@ def deinit(): # optional for Python - `queue_get` (optional): - This method determines whether the custom fuzzer should fuzz the current - queue entry or not + This method determines whether AFL++ should fuzz the current + queue entry or not: all defined custom mutators as well as + all AFL++'s mutators. - `fuzz_count` (optional): @@ -145,12 +146,15 @@ def deinit(): # optional for Python - `fuzz` (optional): - This method performs custom mutations on a given input. It also accepts an - additional test case. Note that this function is optional - but it makes - sense to use it. You would only skip this if `post_process` is used to fix - checksums etc. so if you are using it, e.g., as a post processing library. - Note that a length > 0 *must* be returned! - The returned output buffer is under **your** memory management! + This method performs your custom mutations on a given input. + The add_buf is the contents of another queue item that can be used for + splicing - or anything else - and can also be ignored. If you are not + using this additional data then define `splice_optout` (see above). + This function is optional. + Returing a length of 0 is valid and is interpreted as skipping this + one mutation result. + For non-Python: the returned output buffer is under **your** memory + management! - `describe` (optional): @@ -194,7 +198,7 @@ def deinit(): # optional for Python This method can be used if you want to send data to the target yourself, e.g. via IPC. This replaces some usage of utils/afl_proxy but requires that you start the target with afl-fuzz. - Example: [custom_mutators/examples/custom_send.c](custom_mutators/examples/custom_send.c) + Example: [custom_mutators/examples/custom_send.c](../custom_mutators/examples/custom_send.c) - `queue_new_entry` (optional): @@ -262,6 +266,11 @@ trimmed input. Here's a quick API description: Omitting any of three trimming methods will cause the trimming to be disabled and trigger a fallback to the built-in default trimming routine. +**IMPORTANT** If you have a custom post process mutator that needs to be run +after trimming, you must call it yourself at the end of your successful +trimming! + + ### Environment Variables Optionally, the following environment variables are supported: @@ -373,4 +382,4 @@ See [example.c](../custom_mutators/examples/example.c) and - [bruce30262/libprotobuf-mutator_fuzzing_learning](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator) - [thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator) - [XML Fuzzing@NullCon 2017](https://www.agarri.fr/docs/XML_Fuzzing-NullCon2017-PUBLIC.pdf) - - [A bug detected by AFL + XML-aware mutators](https://bugs.chromium.org/p/chromium/issues/detail?id=930663) \ No newline at end of file + - [A bug detected by AFL + XML-aware mutators](https://bugs.chromium.org/p/chromium/issues/detail?id=930663) diff --git a/docs/env_variables.md b/docs/env_variables.md index b1f23159..b3519107 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -135,6 +135,12 @@ subset of the settings discussed in section 1, with the exception of: - `TMPDIR` and `AFL_KEEP_ASSEMBLY`, since no temporary assembly files are created. + - LLVM modes compiling C++ will normally set rpath in the binary if LLVM is + not in a usual location (/usr or /lib). Setting `AFL_LLVM_NO_RPATH=1` + disables this behaviour in case it isn't desired. For example, the compiling + toolchain might be in a custom location, but the target machine has LLVM + runtime libs in the search path. + Then there are a few specific features that are only available in instrumentation mode: @@ -190,6 +196,19 @@ in the specified file. For more information, see [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md). +#### INJECTIONS + +This feature is able to find simple injection vulnerabilities in insecure +calls to mysql/mariadb/nosql/postgresql/ldap and XSS in libxml2. + + - Setting `AFL_LLVM_INJECTIONS_ALL` will enable all injection hooking + + - Setting `AFL_LLVM_INJECTIONS_SQL` will enable SQL injection hooking + + - Setting `AFL_LLVM_INJECTIONS_LDAP` will enable LDAP injection hooking + + - Setting `AFL_LLVM_INJECTIONS_XSS` will enable XSS injection hooking + #### LAF-INTEL This great feature will split compares into series of single byte comparisons to @@ -229,6 +248,9 @@ use (which only ever the author of this LTO implementation will use). These are used if several separated instrumentations are performed which are then later combined. + - `AFL_LLVM_LTO_CALLER` activates collision free CALLER instrumentation + - `AFL_LLVM_LTO_CALLER` sets the maximum mumber of single block functions + to dig deeper into a real function. Default 0. - `AFL_LLVM_DOCUMENT_IDS=file` will document to a file which edge ID was given to which function. This helps to identify functions with variable bytes or which functions were touched by an input. @@ -327,6 +349,9 @@ checks or alter some of the more exotic semantics of the tool: (`-i in`). This is an important feature to set when resuming a fuzzing session. + - `AFL_IGNORE_SEED_PROBLEMS` will skip over crashes and timeouts in the seeds + instead of exiting. + - Setting `AFL_CRASH_EXITCODE` sets the exit code AFL++ treats as crash. For example, if `AFL_CRASH_EXITCODE='-1'` is set, each input resulting in a `-1` return code (i.e. `exit(-1)` got called), will be treated as if a crash had @@ -356,6 +381,9 @@ checks or alter some of the more exotic semantics of the tool: - Setting `AFL_DISABLE_TRIM` tells afl-fuzz not to trim test cases. This is usually a bad idea! + - Setting `AFL_DISABLE_REDUNDANT` disables any queue items that are redundant. + This can be useful with huge queues. + - Setting `AFL_KEEP_TIMEOUTS` will keep longer running inputs if they reach new coverage @@ -365,6 +393,9 @@ checks or alter some of the more exotic semantics of the tool: - `AFL_EXIT_ON_SEED_ISSUES` will restore the vanilla afl-fuzz behavior which does not allow crashes or timeout seeds in the initial -i corpus. + - `AFL_CRASHING_SEEDS_AS_NEW_CRASH` will treat crashing seeds as new crash. these + crashes will be written to crashes folder as op:dry_run, and orig:<seed_file_name>. + - `AFL_EXIT_ON_TIME` causes afl-fuzz to terminate if no new paths were found within a specified period of time (in seconds). May be convenient for some types of automated jobs. @@ -409,10 +440,15 @@ checks or alter some of the more exotic semantics of the tool: set `AFL_IGNORE_PROBLEMS`. If you additionally want to also ignore coverage from late loaded libraries, you can set `AFL_IGNORE_PROBLEMS_COVERAGE`. - - When running in the `-M` or `-S` mode, setting `AFL_IMPORT_FIRST` causes the - fuzzer to import test cases from other instances before doing anything else. - This makes the "own finds" counter in the UI more accurate. Beyond counter - aesthetics, not much else should change. + - When running with multiple afl-fuzz or with `-F`, setting `AFL_IMPORT_FIRST` + causes the fuzzer to import test cases from other instances before doing + anything else. This makes the "own finds" counter in the UI more accurate. + + - When running with multiple afl-fuzz or with `-F`, setting `AFL_FINAL_SYNC` + will cause the fuzzer to perform a final import of test cases when + terminating. This is beneficial for `-M` main fuzzers to ensure it has all + unique test cases and hence you only need to `afl-cmin` this single + queue. - Setting `AFL_INPUT_LEN_MIN` and `AFL_INPUT_LEN_MAX` are an alternative to the afl-fuzz -g/-G command line option to control the minimum/maximum @@ -514,6 +550,9 @@ checks or alter some of the more exotic semantics of the tool: use a custom afl-qemu-trace or if you need to modify the afl-qemu-trace arguments. + - `AFL_SHA1_FILENAMES` causes AFL++ to generate files named by the SHA1 hash + of their contents, rather than use the standard `id:000000,...` names. + - `AFL_SHUFFLE_QUEUE` randomly reorders the input queue on startup. Requested by some users for unorthodox parallelized fuzzing setups, but not advisable otherwise. @@ -585,7 +624,8 @@ checks or alter some of the more exotic semantics of the tool: Note that this is not a compile time option but a runtime option :-) - Set `AFL_PIZZA_MODE` to 1 to enable the April 1st stats menu, set to -1 - to disable although it is 1st of April. + to disable although it is 1st of April. 0 is the default and means enable + on the 1st of April automatically. - If you need a specific interval to update fuzzer_stats file, you can set `AFL_FUZZER_STATS_UPDATE_INTERVAL` to the interval in seconds you'd @@ -619,6 +659,14 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - Setting `AFL_INST_LIBS` causes the translator to also instrument the code inside any dynamically linked libraries (notably including glibc). + - You can use `AFL_QEMU_INST_RANGES=0xaaaa-0xbbbb,0xcccc-0xdddd` to just + instrument specific memory locations, e.g. a specific library. + Excluding ranges takes priority over any included ranges or `AFL_INST_LIBS`. + + - You can use `AFL_QEMU_EXCLUDE_RANGES=0xaaaa-0xbbbb,0xcccc-0xdddd` to **NOT** + instrument specific memory locations, e.g. a specific library. + Excluding ranges takes priority over any included ranges or `AFL_INST_LIBS`. + - It is possible to set `AFL_INST_RATIO` to skip the instrumentation on some of the basic blocks, which can be useful when dealing with very complex binaries. diff --git a/docs/fuzzing_binary-only_targets.md b/docs/fuzzing_binary-only_targets.md index 9d9d6bb6..a151bce4 100644 --- a/docs/fuzzing_binary-only_targets.md +++ b/docs/fuzzing_binary-only_targets.md @@ -94,8 +94,7 @@ For more information, see In FRIDA mode, you can fuzz binary-only targets as easily as with QEMU mode. FRIDA mode is most of the times slightly faster than QEMU mode. It is also -newer, lacks COMPCOV, and has the advantage that it works on MacOS (both intel -and M1). +newer, and has the advantage that it works on MacOS (both intel and M1). To build FRIDA mode: @@ -113,10 +112,6 @@ The mode is approximately 2-5x slower than compile-time instrumentation, and is less conducive to parallelization. But for binary-only fuzzing, it gives a huge speed improvement if it is possible to use. -If you want to fuzz a binary-only library, then you can fuzz it with frida-gum -via frida_mode/. You will have to write a harness to call the target function in -the library, use afl-frida.c as a template. - You can also perform remote fuzzing with frida, e.g., if you want to fuzz on iPhone or Android devices, for this you can use [https://github.com/ttdennis/fpicker/](https://github.com/ttdennis/fpicker/) as @@ -302,7 +297,6 @@ some are very hard to set up... * S2E: [https://github.com/S2E](https://github.com/S2E) * TinyInst: [https://github.com/googleprojectzero/TinyInst](https://github.com/googleprojectzero/TinyInst) - (Mac/Windows only) * ... please send me any missing that are good ## Closing words diff --git a/docs/fuzzing_in_depth.md b/docs/fuzzing_in_depth.md index f75ca5dc..82437807 100644 --- a/docs/fuzzing_in_depth.md +++ b/docs/fuzzing_in_depth.md @@ -599,32 +599,40 @@ during fuzzing) and their number, a value between 50-500MB is recommended. You can set the cache size (in MB) by setting the environment variable `AFL_TESTCACHE_SIZE`. -There should be one main fuzzer (`-M main-$HOSTNAME` option) and as many -secondary fuzzers (e.g., `-S variant1`) as you have cores that you use. Every -`-M`/`-S` entry needs a unique name (that can be whatever), however, the same -`-o` output directory location has to be used for all instances. +There should be one main fuzzer (`-M main-$HOSTNAME` option - set also +`AFL_FINAL_SYNC=1`) and as many secondary fuzzers (e.g., `-S variant1`) as you +have cores that you use. Every `-M`/`-S` entry needs a unique name (that can be +whatever), however, the same `-o` output directory location has to be used for +all instances. For every secondary fuzzer there should be a variation, e.g.: -* one should fuzz the target that was compiled differently: with sanitizers - activated (`export AFL_USE_ASAN=1 ; export AFL_USE_UBSAN=1 ; export - AFL_USE_CFISAN=1`) +* one should fuzz the target that was compiled with sanitizers activated + (`export AFL_USE_ASAN=1 ; export AFL_USE_UBSAN=1 ; export AFL_USE_CFISAN=1`) * one or two should fuzz the target with CMPLOG/redqueen (see above), at least - one cmplog instance should follow transformations (`-l AT`) + one cmplog instance should follow transformations (`-l 2AT`) * one to three fuzzers should fuzz a target compiled with laf-intel/COMPCOV (see above). Important note: If you run more than one laf-intel/COMPCOV fuzzer and you want them to share their intermediate results, the main fuzzer (`-M`) must - be one of them! (Although this is not really recommended.) - -All other secondaries should be used like this: -* a quarter to a third with the MOpt mutator enabled: `-L 0` -* run with a different power schedule, recommended are: `fast` (default), + be one of them (although this is not really recommended). + +The other secondaries should be run like this: +* 10% with the MOpt mutator enabled: `-L 0` +* 10% should use the old queue cycling with `-Z` +* 50-70% should run with `AFL_DISABLE_TRIM` +* 40% should run with `-P explore` and 20% with `-P exploit` +* If you use `-a` then set 30% of the instances to not use `-a`; if you did + not set `-a` (why??), then set 30% to `-a ascii` and 30% to `-a binary`. +* run each with a different power schedule, recommended are: `fast` (default), `explore`, `coe`, `lin`, `quad`, `exploit`, and `rare` which you can set with the `-p` option, e.g., `-p explore`. See the [FAQ](FAQ.md#what-are-power-schedules) for details. -* a few instances should use the old queue cycling with `-Z` + +It can be useful to set `AFL_IGNORE_SEED_PROBLEMS=1` to skip over seeds that +crash or timeout during startup. Also, it is recommended to set `export AFL_IMPORT_FIRST=1` to load test cases -from other fuzzers in the campaign first. +from other fuzzers in the campaign first. But note that can slow down the start +of the first fuzz by quite a lot of you have many fuzzers and/or many seeds. If you have a large corpus, a corpus from a previous run or are fuzzing in a CI, then also set `export AFL_CMPLOG_ONLY_NEW=1` and `export AFL_FAST_CAL=1`. @@ -940,7 +948,7 @@ too long for your overall available fuzz run time. * 65% for `AFL_DISABLE_TRIM` * 50% for `AFL_KEEP_TIMEOUTS` * 50% use a dictionary generated by `AFL_LLVM_DICT2FILE` + `AFL_LLVM_DICT2FILE_NO_MAIN=1` - * 40% use MOpt (`-L 0`) + * 10% use MOpt (`-L 0`) * 40% for `AFL_EXPAND_HAVOC_NOW` * 20% for old queue processing (`-Z`) * for CMPLOG targets, 70% for `-l 2`, 10% for `-l 3`, 20% for `-l 2AT` @@ -950,7 +958,7 @@ too long for your overall available fuzz run time. campaign but not good for short CI runs. How this can look like can, e.g., be seen at AFL++'s setup in Google's -[oss-fuzz](https://github.com/google/oss-fuzz/blob/master/infra/base-images/base-builder/compile_afl) +[previous oss-fuzz version](https://github.com/google/oss-fuzz/blob/3e2c5312417d1a6f9564472f3df1fd27759b289d/infra/base-images/base-builder/compile_afl) and [clusterfuzz](https://github.com/google/clusterfuzz/blob/master/src/clusterfuzz/_internal/bot/fuzzers/afl/launcher.py). diff --git a/docs/resources/1_instrument_target.drawio.svg b/docs/resources/1_instrument_target.drawio.svg index af6ac397..c93fa2b8 100644 --- a/docs/resources/1_instrument_target.drawio.svg +++ b/docs/resources/1_instrument_target.drawio.svg @@ -1,4 +1,4 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- Do not edit this file with editors other than diagrams.net --> <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> -<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="1041px" height="301px" viewBox="-0.5 -0.5 1041 301" content="<mxfile host="Electron" modified="2022-01-14T14:14:06.979Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/16.1.2 Chrome/96.0.4664.55 Electron/16.0.5 Safari/537.36" etag="SKxyD_wE9pHQQvyJq3sV" version="16.1.2" type="device"><diagram name="1 - instrument target" id="y32N0Cs56pMhbVcY_pYT">7Vttd6I4FP41nrPzgR5efP2oTm27o213nZlt90tPgADZBsKE0Gp//SYQRISK2mo9HT19ITfJDbnPzcPNDTaMoT+7oCD0JsSGuKGr9qxhfG3out7qafyfkMxTSbfTSgUuRXYq0nLBFL1AKVSlNEY2jAoNGSGYobAotEgQQIsVZIBS8lxs5hBcHDUELiwJphbAZek/yGaelGrtXl5xCZHryaG7eietMIH16FISB3K8hm44ySet9kGmS0408oBNnpdExnnDGFJCWHrlz4YQC9tmZruY3/9g19AaOffjqKOi7o19paTKRtt0WcyQwoDtrFo5DxXj7urbX5cPF1SfKt+mPVN2UZ8AjmFmhWSubJ7ZN7EQFErUhjF49hCD0xBYovaZexSXeczHvKTxy4hR8giHBBPKJQEJeLOBHANSBmcrmNXMSFuYmbsvJD5kdM77SS0KB1lOYF70yeclR1CbUugtOYGhSiGQ3ucutOcm5BfSiluApVdYtI2ZsE0IAn7tiuurgFsq9sXs+XIB1IUsa8ZHXW75NjwchPEKGg4J2FSqexUyQC250jVdFM2I4JjB/kKsLm5tMxAr8H8d1049rHo1qnsC1SiB+jf8FSPKYRDwRY9rUNK2XTUN4Z3iI8EaAR9hYYlLiJ8gQxYQiGDkBlxmcdtD3msgzMurcF9W+Mi2Max2AU7FDKBA9EuAXEF7b7g2280CrnqnYr22DwlsswTsTcgQCQB+F2BtEHmLtr8LynpPPTKUWyUUoc1jCFkklHnEFZif59JBkWbzNmNCQgnof5CxuTQpiBk54KJ+E5Zi8muRpBADhp6KAVYVKrLrLUHJg+xV/m72zlpFLRGJqQVlx+WopaTLqNWVPj9LuhJXWcxqd+/p1sdIB3Cn40a81ymj1G7vCHmvWa9sz5hnEccnoowj9xi9rW7kLu+GsHZC+LAIG1rvsAh/CE/DGWJ3ojt/SqWl+6WarzOpOSnMs0LAZ7rUSRTvl+vybkkp6/dbe1One1hvKu8BTwRyWMibvQM/IponAvnE3tRe0bHvkPJDdqG7eNNxbzU6RjGJpHWNHfcZtZr27RHtikdKmgQGIjVCoZNI279iccbAYWXiTKUvbkQfuYh5sXlmEZ8X+qNxiONI/JZKJiamSCozYHKf0kc2sYTYiV9eUOA+oODBhiHzznybEwBQIoihxXiNwjyomDBiCnCwwscJEYZUcQhV0CJ9nbVb5LDlvWbp62mirSGOf9L+S2lukOe4U5lJX5eMv99wLT6xYbnuj6oUu4VB4PJR5P9B8qOKsH/waqq9rOVLTdu1tzz+OXmPezbOuoe76Yvh8OF2/OPi6nq7W3cti3doHfRGea/huH998fqdEu6b9EuJeD88O88EfW+XtG1tTcVb5Gw1rVV8OrYqcrbdipytvreDNK1zfOxoLrFjzoFAnBgoJDk4iNZQYN5iJwYs61H5rcHARoIrVBKUu3DvX6ZeYZz6cYY3k9vhzc+q1RRg4TSS17gY+MKN07+i4vvNl41G8MMxcfc4gETpScx+BaeKUZNHy3AxrrjKWfCN5LH3E6BjY5LV5O3HE4l+HDv3N0XVaUy7rp3Mhsg4bI013j1OfxvLV52ufDDLW0ssH4EAMQ4bXUfsi0a7UvsEzM4SCodFbSqbh0IEgeWVe6FAIMJ/5TQE1XOqBAkxbET1/Wn/uoKeR1eV8nGldFIp/V4p/TFIxCdC3eZlGOPYCDXL4x/TmoVb7EmHaTyUs+WCX0WslO8jtl3F9XpLAZuaRmhmjLAtms8jBv0NFy/3Rwe5Ma3Y9wwn4LFCPIFRMuhADjdNhyu1S3ZNpWVaszCLq/h3WJmGdmwrUy+/z/T5Qp3FW8Z1oU7GU0cS6mSKj4k2bcUnNnLmdZQ5SVo1Kt5V3Y4hKQQMlqIWD9AARlG5vTlPagNBVg4RAZHJHS+JjKDjIAvBwJqfnQKKrWirpX8gbbXvx6Ydxrfqv+r1n8/+3eQFKUrVwlhlrcDuiy8L5NarfPRs/1pyLTksmaVVYZVM9tZjjVIGbsPz7fqDrlVF73eqUQmmfgKzlATZGcvVNykPjOUmmZPPjuXq/mtnLFs1C3zPWG4SGn52LFcffTtjadQs8D1j2TphWdp97YylVrPAd8aSF/NvB6bN869gGuf/Aw==</diagram></mxfile>" style="background-color: rgb(255, 255, 255);"><defs/><g><rect x="0" y="0" width="1040" height="300" fill="rgb(255, 255, 255)" stroke="none" pointer-events="all"/><rect x="400" y="0" width="240" height="30" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 238px; height: 1px; padding-top: 15px; margin-left: 401px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;"><span>Instrument target</span></div></div></div></foreignObject><text x="520" y="19" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Instrument target</text></switch></g><rect x="696" y="260" width="160" height="30" rx="3.6" ry="3.6" fill="none" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 158px; height: 1px; padding-top: 275px; margin-left: 697px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">Required task</div></div></div></foreignObject><text x="776" y="279" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Required task</text></switch></g><rect x="870" y="260" width="160" height="30" rx="3.6" ry="3.6" fill="none" stroke="#000000" stroke-dasharray="3 3" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 158px; height: 1px; padding-top: 275px; margin-left: 871px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">Optional task</div></div></div></foreignObject><text x="950" y="279" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Optional task</text></switch></g><path d="M 400 139.5 L 423.63 139.5" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 428.88 139.5 L 421.88 143 L 423.63 139.5 L 421.88 136 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><path d="M 190 139.66 L 213.63 139.66" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 218.88 139.66 L 211.88 143.16 L 213.63 139.66 L 211.88 136.16 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/><rect x="10" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 11px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#a-selecting-the-best-afl-compiler-for-instrumenting-the-target">Select compiler</a><br /><br />LTO mode<br />(<span>clang/clang++ 11+</span><span>)</span><br /><br />LLVM mode<br />(<span>clang/clang++ 3.8+</span><span>)</span><br /><br />GCC_PLUGIN mode<br />(<span>gcc 5+</span><span>)</span><br /><br />GCC/CLANG mode<br />(other)</div></div></div></foreignObject><text x="100" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Select compiler...</text></switch></g><rect x="220" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" stroke-dasharray="3 3" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 221px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#b-selecting-instrumentation-options">Select options</a><br /><br />Select options depending on<br />the compiler:<br /><br />COMPCOV<br />(only LLVM & LTO)<br /><br />CmpLog<br />(only LLVM & LTO)<br /><br />selective instrumentation<br />(LTO, LLVM, GCC_PLUGIN)</div></div></div></foreignObject><text x="310" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Select options...</text></switch></g><path d="M 610 140 L 630 140 L 620 140 L 633.63 140" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 638.88 140 L 631.88 143.5 L 633.63 140 L 631.88 136.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/><rect x="430" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" stroke-dasharray="3 3" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 431px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#c-selecting-sanitizers">Select sanitizer</a><br /><br />Max. one sanitizer type each<br />in a fuzzing campaign:<br /><br />ASAN<br />CFISAN<br />LSAN<br />MSAN<br />TSAN<br />UBSAN</div></div></div></foreignObject><text x="520" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Select sanitizer...</text></switch></g><rect x="850" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 851px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#e-instrumenting-the-target">Compile target source code</a><br /><br />Compile target source code depending on the build system:<br /><br />configure<br />CMake<br />Meson Build System<br />other</div></div></div></foreignObject><text x="940" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Compile target source code...</text></switch></g><path d="M 820 140 L 840 140 L 830 140 L 843.63 140" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 848.88 140 L 841.88 143.5 L 843.63 140 L 841.88 136.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/><rect x="640" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" stroke-dasharray="3 3" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 641px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#d-modifying-the-target">Modify target</a><br /><br />Create a fuzzing harness<br />by hand for better efficiency.</div></div></div></foreignObject><text x="730" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Modify target...</text></switch></g><path d="M 10 68 L 190 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 220 68 L 400 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 430 68 L 610 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 640 68 L 820 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 850 68 L 1030 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/></g><switch><g requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"/><a transform="translate(0,-5)" xlink:href="https://www.diagrams.net/doc/faq/svg-export-text-problems" target="_blank"><text text-anchor="middle" font-size="10px" x="50%" y="100%">Viewer does not support full SVG 1.1</text></a></switch></svg> \ No newline at end of file +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="1041px" height="301px" viewBox="-0.5 -0.5 1041 301" content="<mxfile host="Electron" modified="2022-01-14T14:14:06.979Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/16.1.2 Chrome/96.0.4664.55 Electron/16.0.5 Safari/537.36" etag="SKxyD_wE9pHQQvyJq3sV" version="16.1.2" type="device"><diagram name="1 - instrument target" id="y32N0Cs56pMhbVcY_pYT">7Vttd6I4FP41nrPzgR5efP2oTm27o213nZlt90tPgADZBsKE0Gp//SYQRISK2mo9HT19ITfJDbnPzcPNDTaMoT+7oCD0JsSGuKGr9qxhfG3out7qafyfkMxTSbfTSgUuRXYq0nLBFL1AKVSlNEY2jAoNGSGYobAotEgQQIsVZIBS8lxs5hBcHDUELiwJphbAZek/yGaelGrtXl5xCZHryaG7eietMIH16FISB3K8hm44ySet9kGmS0408oBNnpdExnnDGFJCWHrlz4YQC9tmZruY3/9g19AaOffjqKOi7o19paTKRtt0WcyQwoDtrFo5DxXj7urbX5cPF1SfKt+mPVN2UZ8AjmFmhWSubJ7ZN7EQFErUhjF49hCD0xBYovaZexSXeczHvKTxy4hR8giHBBPKJQEJeLOBHANSBmcrmNXMSFuYmbsvJD5kdM77SS0KB1lOYF70yeclR1CbUugtOYGhSiGQ3ucutOcm5BfSiluApVdYtI2ZsE0IAn7tiuurgFsq9sXs+XIB1IUsa8ZHXW75NjwchPEKGg4J2FSqexUyQC250jVdFM2I4JjB/kKsLm5tMxAr8H8d1049rHo1qnsC1SiB+jf8FSPKYRDwRY9rUNK2XTUN4Z3iI8EaAR9hYYlLiJ8gQxYQiGDkBlxmcdtD3msgzMurcF9W+Mi2Max2AU7FDKBA9EuAXEF7b7g2280CrnqnYr22DwlsswTsTcgQCQB+F2BtEHmLtr8LynpPPTKUWyUUoc1jCFkklHnEFZif59JBkWbzNmNCQgnof5CxuTQpiBk54KJ+E5Zi8muRpBADhp6KAVYVKrLrLUHJg+xV/m72zlpFLRGJqQVlx+WopaTLqNWVPj9LuhJXWcxqd+/p1sdIB3Cn40a81ymj1G7vCHmvWa9sz5hnEccnoowj9xi9rW7kLu+GsHZC+LAIG1rvsAh/CE/DGWJ3ojt/SqWl+6WarzOpOSnMs0LAZ7rUSRTvl+vybkkp6/dbe1One1hvKu8BTwRyWMibvQM/IponAvnE3tRe0bHvkPJDdqG7eNNxbzU6RjGJpHWNHfcZtZr27RHtikdKmgQGIjVCoZNI279iccbAYWXiTKUvbkQfuYh5sXlmEZ8X+qNxiONI/JZKJiamSCozYHKf0kc2sYTYiV9eUOA+oODBhiHzznybEwBQIoihxXiNwjyomDBiCnCwwscJEYZUcQhV0CJ9nbVb5LDlvWbp62mirSGOf9L+S2lukOe4U5lJX5eMv99wLT6xYbnuj6oUu4VB4PJR5P9B8qOKsH/waqq9rOVLTdu1tzz+OXmPezbOuoe76Yvh8OF2/OPi6nq7W3cti3doHfRGea/huH998fqdEu6b9EuJeD88O88EfW+XtG1tTcVb5Gw1rVV8OrYqcrbdipytvreDNK1zfOxoLrFjzoFAnBgoJDk4iNZQYN5iJwYs61H5rcHARoIrVBKUu3DvX6ZeYZz6cYY3k9vhzc+q1RRg4TSS17gY+MKN07+i4vvNl41G8MMxcfc4gETpScx+BaeKUZNHy3AxrrjKWfCN5LH3E6BjY5LV5O3HE4l+HDv3N0XVaUy7rp3Mhsg4bI013j1OfxvLV52ufDDLW0ssH4EAMQ4bXUfsi0a7UvsEzM4SCodFbSqbh0IEgeWVe6FAIMJ/5TQE1XOqBAkxbET1/Wn/uoKeR1eV8nGldFIp/V4p/TFIxCdC3eZlGOPYCDXL4x/TmoVb7EmHaTyUs+WCX0WslO8jtl3F9XpLAZuaRmhmjLAtms8jBv0NFy/3Rwe5Ma3Y9wwn4LFCPIFRMuhADjdNhyu1S3ZNpWVaszCLq/h3WJmGdmwrUy+/z/T5Qp3FW8Z1oU7GU0cS6mSKj4k2bcUnNnLmdZQ5SVo1Kt5V3Y4hKQQMlqIWD9AARlG5vTlPagNBVg4RAZHJHS+JjKDjIAvBwJqfnQKKrWirpX8gbbXvx6Ydxrfqv+r1n8/+3eQFKUrVwlhlrcDuiy8L5NarfPRs/1pyLTksmaVVYZVM9tZjjVIGbsPz7fqDrlVF73eqUQmmfgKzlATZGcvVNykPjOUmmZPPjuXq/mtnLFs1C3zPWG4SGn52LFcffTtjadQs8D1j2TphWdp97YylVrPAd8aSF/NvB6bN869gGuf/Aw==</diagram></mxfile>" style="background-color: rgb(255, 255, 255);"><defs/><g><rect x="0" y="0" width="1040" height="300" fill="rgb(255, 255, 255)" stroke="none" pointer-events="all"/><rect x="400" y="0" width="240" height="30" fill="none" stroke="none" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 238px; height: 1px; padding-top: 15px; margin-left: 401px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;"><span>Instrument target</span></div></div></div></foreignObject><text x="520" y="19" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Instrument target</text></switch></g><rect x="696" y="260" width="160" height="30" rx="3.6" ry="3.6" fill="none" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 158px; height: 1px; padding-top: 275px; margin-left: 697px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">Required task</div></div></div></foreignObject><text x="776" y="279" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Required task</text></switch></g><rect x="870" y="260" width="160" height="30" rx="3.6" ry="3.6" fill="none" stroke="#000000" stroke-dasharray="3 3" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 158px; height: 1px; padding-top: 275px; margin-left: 871px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">Optional task</div></div></div></foreignObject><text x="950" y="279" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Optional task</text></switch></g><path d="M 400 139.5 L 423.63 139.5" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 428.88 139.5 L 421.88 143 L 423.63 139.5 L 421.88 136 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="all"/><path d="M 190 139.66 L 213.63 139.66" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 218.88 139.66 L 211.88 143.16 L 213.63 139.66 L 211.88 136.16 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/><rect x="10" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 11px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#a-selecting-the-best-afl-compiler-for-instrumenting-the-target">Select compiler</a><br /><br />LTO mode<br />(<span>clang/clang++ 12+</span><span>)</span><br /><br />LLVM mode<br />(<span>clang/clang++ 3.8+</span><span>)</span><br /><br />GCC_PLUGIN mode<br />(<span>gcc 5+</span><span>)</span><br /><br />GCC/CLANG mode<br />(other)</div></div></div></foreignObject><text x="100" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Select compiler...</text></switch></g><rect x="220" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" stroke-dasharray="3 3" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 221px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#b-selecting-instrumentation-options">Select options</a><br /><br />Select options depending on<br />the compiler:<br /><br />COMPCOV<br />(only LLVM & LTO)<br /><br />CmpLog<br />(only LLVM & LTO)<br /><br />selective instrumentation<br />(LTO, LLVM, GCC_PLUGIN)</div></div></div></foreignObject><text x="310" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Select options...</text></switch></g><path d="M 610 140 L 630 140 L 620 140 L 633.63 140" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 638.88 140 L 631.88 143.5 L 633.63 140 L 631.88 136.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/><rect x="430" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" stroke-dasharray="3 3" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 431px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#c-selecting-sanitizers">Select sanitizer</a><br /><br />Max. one sanitizer type each<br />in a fuzzing campaign:<br /><br />ASAN<br />CFISAN<br />LSAN<br />MSAN<br />TSAN<br />UBSAN</div></div></div></foreignObject><text x="520" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Select sanitizer...</text></switch></g><rect x="850" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 851px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#e-instrumenting-the-target">Compile target source code</a><br /><br />Compile target source code depending on the build system:<br /><br />configure<br />CMake<br />Meson Build System<br />other</div></div></div></foreignObject><text x="940" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Compile target source code...</text></switch></g><path d="M 820 140 L 840 140 L 830 140 L 843.63 140" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 848.88 140 L 841.88 143.5 L 843.63 140 L 841.88 136.5 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="all"/><rect x="640" y="40" width="180" height="200" rx="9" ry="9" fill="none" stroke="#000000" stroke-dasharray="3 3" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility" style="overflow: visible; text-align: left;"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 178px; height: 1px; padding-top: 47px; margin-left: 641px;"><div data-drawio-colors="color: rgb(0, 0, 0); " style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><a href="https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/fuzzing_in_depth.md#d-modifying-the-target">Modify target</a><br /><br />Create a fuzzing harness<br />by hand for better efficiency.</div></div></div></foreignObject><text x="730" y="59" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Modify target...</text></switch></g><path d="M 10 68 L 190 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 220 68 L 400 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 430 68 L 610 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 640 68 L 820 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 850 68 L 1030 68" fill="none" stroke="rgb(0, 0, 0)" stroke-miterlimit="10" pointer-events="stroke"/></g><switch><g requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"/><a transform="translate(0,-5)" xlink:href="https://www.diagrams.net/doc/faq/svg-export-text-problems" target="_blank"><text text-anchor="middle" font-size="10px" x="50%" y="100%">Viewer does not support full SVG 1.1</text></a></switch></svg> \ No newline at end of file diff --git a/docs/third_party_tools.md b/docs/third_party_tools.md index 02a40ce5..64a37c83 100644 --- a/docs/third_party_tools.md +++ b/docs/third_party_tools.md @@ -5,6 +5,10 @@ * [afl-rs](https://github.com/rust-fuzz/afl.rs) - AFL++ for RUST * [WASM](https://github.com/fgsect/WAFL) - AFL++ for WASM +## Starting multiple AFL++ instances in parallel with recommended settings: +* [https://github.com/0xricksanchez/AFL_Runner](https://github.com/0xricksanchez/AFL_Runner) +* [https://github.com/MegaManSec/AFLplusplus-Parallel-Gen](https://github.com/MegaManSec/AFLplusplus-Parallel-Gen) + ## Speeding up fuzzing * [libfiowrapper](https://github.com/marekzmyslowski/libfiowrapper) - if the diff --git a/docs/tutorials.md b/docs/tutorials.md index 342080fd..f9378232 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -8,6 +8,7 @@ Here are some good write-ups to show how to effectively use AFL++: * [https://aflplus.plus/docs/tutorials/libxml2_tutorial/](https://aflplus.plus/docs/tutorials/libxml2_tutorial/) * [https://bananamafia.dev/post/gb-fuzz/](https://bananamafia.dev/post/gb-fuzz/) +* [https://bushido-sec.com/index.php/2023/06/19/the-art-of-fuzzing/](https://bushido-sec.com/index.php/2023/06/19/the-art-of-fuzzing/) * [https://securitylab.github.com/research/fuzzing-challenges-solutions-1](https://securitylab.github.com/research/fuzzing-challenges-solutions-1) * [https://securitylab.github.com/research/fuzzing-software-2](https://securitylab.github.com/research/fuzzing-software-2) * [https://securitylab.github.com/research/fuzzing-sockets-FTP](https://securitylab.github.com/research/fuzzing-sockets-FTP) @@ -20,6 +21,13 @@ training, then we can highly recommend the following: * [https://github.com/antonio-morales/Fuzzing101](https://github.com/antonio-morales/Fuzzing101) +A good workflow overview (like our [fuzzing_in_depth.md](fuzzing_in_depth.md)): +* [https://appsec.guide/docs/fuzzing/c-cpp/aflpp/](https://appsec.guide/docs/fuzzing/c-cpp/aflpp/) + +Here is a good workflow description (and tutorial) for qemu_mode: + +* [https://airbus-seclab.github.io/AFLplusplus-blogpost/](https://airbus-seclab.github.io/AFLplusplus-blogpost/) + Here is good workflow description for frida_mode: * [https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html](https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html) @@ -36,6 +44,9 @@ structure is), these links have you covered (some are outdated though): * Superion for AFL++: [https://github.com/adrian-rt/superion-mutator](https://github.com/adrian-rt/superion-mutator) +For a very in-depth explanation on how AFL++ works check out: +[https://blog.ritsec.club/posts/afl-under-hood/](https://blog.ritsec.club/posts/afl-under-hood/) + ## Video Tutorials * [Install AFL++ Ubuntu](https://www.youtube.com/watch?v=5dCvhkbi3RA) diff --git a/frida_mode/Scripting.md b/frida_mode/Scripting.md index dfd09e7b..653687f0 100644 --- a/frida_mode/Scripting.md +++ b/frida_mode/Scripting.md @@ -390,7 +390,7 @@ Consider the [following](test/js/test2.c) test code... -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/include/entry.h b/frida_mode/include/entry.h index edc41467..949fab71 100644 --- a/frida_mode/include/entry.h +++ b/frida_mode/include/entry.h @@ -14,8 +14,6 @@ void entry_init(void); void entry_start(void); -void entry_prologue(GumStalkerIterator *iterator, GumStalkerOutput *output); - void entry_on_fork(void); #endif diff --git a/frida_mode/src/entry.c b/frida_mode/src/entry.c index 05af7ebb..fc49b4d7 100644 --- a/frida_mode/src/entry.c +++ b/frida_mode/src/entry.c @@ -78,31 +78,7 @@ void entry_init(void) { void entry_start(void) { - if (persistent_start == 0) { - - ranges_exclude(); - stalker_trust(); - - } - - if (entry_point == 0) { entry_launch(); } - -} - -static void entry_callout(GumCpuContext *cpu_context, gpointer user_data) { - - UNUSED_PARAMETER(cpu_context); - UNUSED_PARAMETER(user_data); - entry_compiled = TRUE; - entry_launch(); - -} - -void entry_prologue(GumStalkerIterator *iterator, GumStalkerOutput *output) { - - UNUSED_PARAMETER(output); FVERBOSE("AFL_ENTRYPOINT reached"); - if (persistent_start == 0) { ranges_exclude(); @@ -110,7 +86,7 @@ void entry_prologue(GumStalkerIterator *iterator, GumStalkerOutput *output) { } - gum_stalker_iterator_put_callout(iterator, entry_callout, NULL, NULL); + entry_launch(); } diff --git a/frida_mode/src/instrument/instrument.c b/frida_mode/src/instrument/instrument.c index a6aac666..db73d845 100644 --- a/frida_mode/src/instrument/instrument.c +++ b/frida_mode/src/instrument/instrument.c @@ -169,7 +169,6 @@ static void instrument_basic_block(GumStalkerIterator *iterator, if (unlikely(begin)) { instrument_debug_start(instr->address, output); } - if (instr->address == entry_point) { entry_prologue(iterator, output); } if (instr->address == persistent_start) { persistent_prologue(output); } if (instr->address == persistent_ret) { persistent_epilogue(output); } diff --git a/frida_mode/src/instrument/instrument_arm32.c b/frida_mode/src/instrument/instrument_arm32.c index 51f78a35..c1e3f187 100644 --- a/frida_mode/src/instrument/instrument_arm32.c +++ b/frida_mode/src/instrument/instrument_arm32.c @@ -15,7 +15,7 @@ static GHashTable *coverage_blocks = NULL; extern __thread guint64 instrument_previous_pc; -__attribute__((aligned(0x1000))) static guint8 area_ptr_dummy[MAP_SIZE]; +__attribute__((aligned(0x1000))) static guint8 area_ptr_dummy[MAP_INITIAL_SIZE]; #pragma pack(push, 1) typedef struct { diff --git a/frida_mode/src/instrument/instrument_arm64.c b/frida_mode/src/instrument/instrument_arm64.c index 4372861d..1147275f 100644 --- a/frida_mode/src/instrument/instrument_arm64.c +++ b/frida_mode/src/instrument/instrument_arm64.c @@ -22,7 +22,7 @@ gboolean instrument_cache_enabled = FALSE; gsize instrument_cache_size = 0; static GHashTable *coverage_blocks = NULL; -__attribute__((aligned(0x1000))) static guint8 area_ptr_dummy[MAP_SIZE]; +__attribute__((aligned(0x1000))) static guint8 area_ptr_dummy[MAP_INITIAL_SIZE]; #pragma pack(push, 1) typedef struct { @@ -76,6 +76,45 @@ typedef struct { } afl_log_code_asm_t; +typedef struct { + + uint32_t b_imm8; /* br #XX (end) */ + + uint32_t restoration_prolog; /* ldp x16, x17, [sp], #0x90 */ + + uint32_t stp_x0_x1; /* stp x0, x1, [sp, #-0xa0] */ + + uint32_t ldr_x0_p_prev_loc_1; /* ldr x0, #0xXXXX */ + uint32_t ldr_x1_ptr_x0; /* ldr x1, [x0] */ + + uint32_t ldr_x0_p_area_offset; /* ldr x0, #0xXXXX */ + uint32_t eor_x0_x1_x0; /* eor x0, x1, x0 */ + uint32_t ldr_x1_p_area_ptr; /* ldr x1, #0xXXXX */ + uint32_t add_x0_x1_x0; /* add x0, x1, x0 */ + + uint32_t ldrb_w1_x0; /* ldrb w1, [x0] */ + uint32_t add_w1_w1_1; /* add w1, w1, #1 */ + uint32_t add_w1_w1_w1_lsr_8; /* add x1, x1, x1, lsr #8 */ + + uint32_t strb_w1_ptr_x0; /* strb w1, [x0] */ + + uint32_t ldr_x0_p_prev_loc_2; /* ldr x0, #0xXXXX */ + uint32_t ldr_x1_p_area_offset_ror; /* ldr x1, #0xXXXX */ + uint32_t str_x1_ptr_x0; /* str x1, [x0] */ + + uint32_t ldp_x0_x1; /* ldp x0, x1, [sp, #-0xa0] */ + + uint32_t b_end; /* skip the data */ + + uint64_t area_ptr; + uint64_t prev_loc_ptr; + uint64_t area_offset; + uint64_t area_offset_ror; + + uint8_t end[0]; + +} afl_log_code_asm_long_t; + #pragma pack(pop) typedef union { @@ -85,6 +124,13 @@ typedef union { } afl_log_code; +typedef union { + + afl_log_code_asm_long_t code; + uint8_t bytes[0]; + +} afl_log_code_long; + static const afl_log_code_asm_t template = { @@ -119,6 +165,46 @@ static const afl_log_code_asm_t template = ; +static const afl_log_code_asm_long_t template_long = + {.b_imm8 = 0x1400001a, + + .restoration_prolog = 0xa8c947f0, /* ldp x16, x17, [sp], #0x90 */ + + .stp_x0_x1 = 0xa93607e0, /* stp x0, x1, [sp, #-0xa0] */ + + .ldr_x0_p_prev_loc_1 = 0x58000220, /* ldr x0, #0xXXXX */ + .ldr_x1_ptr_x0 = 0xf9400001, /* ldr x1, [x0] */ + + .ldr_x0_p_area_offset = 0x58000220, /* ldr x0, #0xXXXX */ + .eor_x0_x1_x0 = 0xca000020, /* eor x0, x1, x0 */ + .ldr_x1_p_area_ptr = 0x58000161, /* ldr x1, #0xXXXX */ + .add_x0_x1_x0 = 0x8b000020, /* add x0, x1, x0 */ + + .ldrb_w1_x0 = 0x39400001, /* ldrb w1, [x0] */ + .add_w1_w1_1 = 0x11000421, /* add w1, w1, #1 */ + .add_w1_w1_w1_lsr_8 = 0x8b412021, /* add x1, x1, x1, lsr #8 */ + + .strb_w1_ptr_x0 = 0x39000001, /* strb w1, [x0] */ + + .ldr_x0_p_prev_loc_2 = 0x580000e0, /* ldr x0, #0xXXXX */ + .ldr_x1_p_area_offset_ror = 0x58000141, /* ldr x1, #0xXXXX */ + .str_x1_ptr_x0 = 0xf9000001, /* str x1, [x0] */ + + .ldp_x0_x1 = 0xa97607e0, /* ldp x0, x1, [sp, #-0xa0] */ + + .b_end = 0x14000009, /* skip the data */ + + .area_ptr = 0x0, + .prev_loc_ptr = 0x0, + .area_offset = 0x0, + .area_offset_ror = 0x0, + + .end = {} + +} + +; + gboolean instrument_is_coverage_optimize_supported(void) { return true; @@ -266,16 +352,22 @@ static gboolean instrument_coverage_in_range(gssize offset) { } -static void instrument_patch_ardp(guint32 *patch, GumAddress insn, +static bool instrument_patch_ardp(guint32 *patch, GumAddress insn, GumAddress target) { - if (!PAGE_ALIGNED(target)) { FATAL("Target not page aligned"); } + if (!PAGE_ALIGNED(target)) { + + FWARNF("Target not page aligned"); + return false; + + } gssize distance = target - (GUM_ADDRESS(insn) & PAGE_MASK); if (!instrument_coverage_in_range(distance)) { - FATAL("Patch out of range 0x%016lX->0x%016lX = 0x%016lX", insn, target, - distance); + FVERBOSE("Patch out of range 0x%016lX->0x%016lX = 0x%016lX", insn, target, + distance); + return false; } @@ -283,6 +375,103 @@ static void instrument_patch_ardp(guint32 *patch, GumAddress insn, guint32 imm_high = ((distance >> 14) & 0x7FFFF) << 5; *patch |= imm_low; *patch |= imm_high; + return true; + +} + +bool instrument_write_inline(GumArm64Writer *cw, GumAddress code_addr, + guint64 area_offset, gsize area_offset_ror) { + + afl_log_code code = {0}; + code.code = template; + + /* + * Given our map is allocated on a 64KB boundary and our map is a multiple of + * 64KB in size, then it should also end on a 64 KB boundary. It is followed + * by our previous_pc, so this too should be 64KB aligned. + */ + g_assert(PAGE_ALIGNED(instrument_previous_pc_addr)); + g_assert(PAGE_ALIGNED(__afl_area_ptr)); + + if (!instrument_patch_ardp( + &code.code.adrp_x0_prev_loc1, + code_addr + offsetof(afl_log_code, code.adrp_x0_prev_loc1), + GUM_ADDRESS(instrument_previous_pc_addr))) { + + return false; + + } + + /* + * The mov instruction supports up to a 16-bit offset. If our offset is out of + * range, then it can end up clobbering the op-code portion of the instruction + * rather than just the operands. So return false and fall back to the + * alternative instrumentation. + */ + if (area_offset > UINT16_MAX) { return false; } + + code.code.mov_x0_curr_loc |= area_offset << 5; + + if (!instrument_patch_ardp( + &code.code.adrp_x1_area_ptr, + code_addr + offsetof(afl_log_code, code.adrp_x1_area_ptr), + GUM_ADDRESS(__afl_area_ptr))) { + + return false; + + } + + if (!instrument_patch_ardp( + &code.code.adrp_x0_prev_loc2, + code_addr + offsetof(afl_log_code, code.adrp_x0_prev_loc2), + GUM_ADDRESS(instrument_previous_pc_addr))) { + + return false; + + } + + code.code.mov_x1_curr_loc_shr_1 |= (area_offset_ror << 5); + + if (instrument_suppress) { + + gum_arm64_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code)); + + } else { + + size_t offset = offsetof(afl_log_code, code.stp_x0_x1); + gum_arm64_writer_put_bytes(cw, &code.bytes[offset], + sizeof(afl_log_code) - offset); + + } + + return true; + +} + +bool instrument_write_inline_long(GumArm64Writer *cw, GumAddress code_addr, + guint64 area_offset, gsize area_offset_ror) { + + afl_log_code_long code = {0}; + code.code = template_long; + + code.code.area_ptr = GUM_ADDRESS(__afl_area_ptr); + code.code.prev_loc_ptr = GUM_ADDRESS(instrument_previous_pc_addr); + code.code.area_offset = area_offset; + code.code.area_offset_ror = GUM_ADDRESS(area_offset_ror); + + if (instrument_suppress) { + + gum_arm64_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code_long)); + + } else { + + size_t offset = offsetof(afl_log_code_long, code.stp_x0_x1); + gum_arm64_writer_put_bytes(cw, &code.bytes[offset], + sizeof(afl_log_code_long) - offset); + + } + + return true; } @@ -312,6 +501,8 @@ void instrument_coverage_optimize(const cs_insn *instr, } // gum_arm64_writer_put_brk_imm(cw, 0x0); + // uint32_t jmp_dot = 0x14000000; + // gum_arm64_writer_put_bytes(cw, (guint8 *)&jmp_dot, sizeof(jmp_dot)); if (instrument_suppress) { instrument_coverage_suppress_init(); } @@ -343,47 +534,19 @@ void instrument_coverage_optimize(const cs_insn *instr, } - code.code = template; - - /* - * Given our map is allocated on a 64KB boundary and our map is a multiple of - * 64KB in size, then it should also end on a 64 KB boundary. It is followed - * by our previous_pc, so this too should be 64KB aligned. - */ - g_assert(PAGE_ALIGNED(instrument_previous_pc_addr)); - g_assert(PAGE_ALIGNED(__afl_area_ptr)); - - instrument_patch_ardp( - &code.code.adrp_x0_prev_loc1, - code_addr + offsetof(afl_log_code, code.adrp_x0_prev_loc1), - GUM_ADDRESS(instrument_previous_pc_addr)); - - code.code.mov_x0_curr_loc |= area_offset << 5; - - instrument_patch_ardp( - &code.code.adrp_x1_area_ptr, - code_addr + offsetof(afl_log_code, code.adrp_x1_area_ptr), - GUM_ADDRESS(__afl_area_ptr)); - map_size_pow2 = util_log2(__afl_map_size); area_offset_ror = util_rotate(area_offset, 1, map_size_pow2); - instrument_patch_ardp( - &code.code.adrp_x0_prev_loc2, - code_addr + offsetof(afl_log_code, code.adrp_x0_prev_loc2), - GUM_ADDRESS(instrument_previous_pc_addr)); - - code.code.mov_x1_curr_loc_shr_1 |= (area_offset_ror << 5); + code.code = template; - if (instrument_suppress) { + if (!instrument_write_inline(cw, code_addr, area_offset, area_offset_ror)) { - gum_arm64_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code)); + if (!instrument_write_inline_long(cw, code_addr, area_offset, + area_offset_ror)) { - } else { + FATAL("Failed to write inline instrumentation"); - size_t offset = offsetof(afl_log_code, code.stp_x0_x1); - gum_arm64_writer_put_bytes(cw, &code.bytes[offset], - sizeof(afl_log_code) - offset); + } } diff --git a/frida_mode/src/instrument/instrument_x64.c b/frida_mode/src/instrument/instrument_x64.c index 8338f8e7..3983c3ba 100644 --- a/frida_mode/src/instrument/instrument_x64.c +++ b/frida_mode/src/instrument/instrument_x64.c @@ -58,6 +58,7 @@ typedef union { } jcc_insn; static GHashTable *coverage_blocks = NULL; +static GHashTable *coverage_blocks_long = NULL; gboolean instrument_is_coverage_optimize_supported(void) { @@ -127,6 +128,64 @@ typedef struct { } afl_log_code_asm_t; +typedef struct { + + // cur_location = (block_address >> 4) ^ (block_address << 8); + // shared_mem[cur_location ^ prev_location]++; + // prev_location = cur_location >> 1; + + // mov QWORD PTR [rsp-0x88],rax + // lahf + // mov QWORD PTR [rsp-0x90],rax + // mov QWORD PTR [rsp-0x98],rbx + + // mov rax, 0xXXXXXXXXXXXXXXXXX /* p_prev_loc */ + // mov eax, dword ptr [rax] /* prev_loc */ + // xor eax,0x3f77 /* cur_loc */ + + // mov rbx, 0xXXXXXXXXXXXXXXXXX /* map */ + // add rax,rbx + + // mov bl,BYTE PTR [rax] + // add bl,0x1 + // adc bl,0x0 + // mov BYTE PTR [rax],bl + + // mov rax, 0xXXXXXXXXXXXXXXXXX /* p_prev_loc */ + // mov dword ptr [rax], 0xXXXXXXXXX /* prev_loc */ + + // mov rbx,QWORD PTR [rsp-0x98] + // mov rax,QWORD PTR [rsp-0x90] + // sahf + // mov rax,QWORD PTR [rsp-0x88] + + uint8_t mov_rax_rsp_88[8]; + uint8_t lahf; + uint8_t mov_rax_rsp_90[8]; + uint8_t mov_rbx_rsp_98[8]; + + uint8_t mov_rax_prev_loc_ptr1[10]; + uint8_t mov_eax_prev_loc[2]; + uint8_t xor_eax_curr_loc[5]; + + uint8_t mov_rbx_map_ptr[10]; + uint8_t add_rax_rbx[3]; + + uint8_t mov_rbx_ptr_rax[2]; + uint8_t add_bl_1[3]; + uint8_t adc_bl_0[3]; + uint8_t mov_ptr_rax_rbx[2]; + + uint8_t mov_rax_prev_loc_ptr2[10]; + uint8_t mov_prev_loc_curr_loc_shr1[6]; + + uint8_t mov_rsp_98_rbx[8]; + uint8_t mov_rsp_90_rax[8]; + uint8_t sahf; + uint8_t mov_rsp_88_rax[8]; + +} afl_log_code_asm_long_t; + #pragma pack(pop) static const afl_log_code_asm_t template = @@ -158,6 +217,41 @@ static const afl_log_code_asm_t template = ; +static const afl_log_code_asm_long_t template_long = + { + + .mov_rax_rsp_88 = {0x48, 0x89, 0x84, 0x24, 0x78, 0xFF, 0xFF, 0xFF}, + .lahf = 0x9f, + .mov_rax_rsp_90 = {0x48, 0x89, 0x84, 0x24, 0x70, 0xFF, 0xFF, 0xFF}, + .mov_rbx_rsp_98 = {0x48, 0x89, 0x9C, 0x24, 0x68, 0xFF, 0xFF, 0xFF}, + + .mov_rax_prev_loc_ptr1 = {0x48, 0xB8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF}, + .mov_eax_prev_loc = {0x8b, 0x00}, + .xor_eax_curr_loc = {0x35}, + + .mov_rbx_map_ptr = {0x48, 0xBB, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF}, + .add_rax_rbx = {0x48, 0x01, 0xd8}, + + .mov_rbx_ptr_rax = {0x8a, 0x18}, + .add_bl_1 = {0x80, 0xc3, 0x01}, + .adc_bl_0 = {0x80, 0xd3, 0x00}, + .mov_ptr_rax_rbx = {0x88, 0x18}, + + .mov_rax_prev_loc_ptr2 = {0x48, 0xB8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF}, + .mov_prev_loc_curr_loc_shr1 = {0xc7, 0x00, 0xFF, 0xFF, 0xFF, 0xFF}, + + .mov_rsp_98_rbx = {0x48, 0x8B, 0x9C, 0x24, 0x68, 0xFF, 0xFF, 0xFF}, + .mov_rsp_90_rax = {0x48, 0x8B, 0x84, 0x24, 0x70, 0xFF, 0xFF, 0xFF}, + .sahf = 0x9e, + .mov_rsp_88_rax = {0x48, 0x8B, 0x84, 0x24, 0x78, 0xFF, 0xFF, 0xFF}, + +} + +; + typedef union { afl_log_code_asm_t code; @@ -165,6 +259,13 @@ typedef union { } afl_log_code; +typedef union { + + afl_log_code_asm_long_t code; + uint8_t bytes[0]; + +} afl_log_code_long; + void instrument_coverage_optimize_init(void) { FVERBOSE("__afl_area_ptr: %p", __afl_area_ptr); @@ -182,16 +283,19 @@ static void instrument_coverage_switch_insn(GumStalkerObserver *self, cs_x86 *x86; cs_x86_op *op; + bool is_short = false; + bool is_long = false; + if (from_insn == NULL) { return; } x86 = &from_insn->detail->x86; op = x86->operands; - if (!g_hash_table_contains(coverage_blocks, GSIZE_TO_POINTER(*target))) { - - return; + is_short = g_hash_table_contains(coverage_blocks, GSIZE_TO_POINTER(*target)); + is_long = + g_hash_table_contains(coverage_blocks_long, GSIZE_TO_POINTER(*target)); - } + if (!is_short && !is_long) { return; } switch (from_insn->id) { @@ -212,15 +316,41 @@ static void instrument_coverage_switch_insn(GumStalkerObserver *self, break; case X86_INS_RET: - instrument_cache_insert(start_address, - (guint8 *)*target + sizeof(afl_log_code)); + if (is_short) { + + instrument_cache_insert(start_address, + (guint8 *)*target + sizeof(afl_log_code)); + + } else if (is_long) { + + instrument_cache_insert(start_address, + (guint8 *)*target + sizeof(afl_log_code_long)); + + } else { + + FATAL("Something has gone wrong here!"); + + } + break; default: return; } - *target = (guint8 *)*target + sizeof(afl_log_code); + if (is_short) { + + *target = (guint8 *)*target + sizeof(afl_log_code); + + } else if (is_long) { + + *target = (guint8 *)*target + sizeof(afl_log_code_long); + + } else { + + FATAL("Something has gone wrong here!"); + + } } @@ -270,22 +400,22 @@ static void instrument_coverage_suppress_init(void) { } + coverage_blocks_long = g_hash_table_new(g_direct_hash, g_direct_equal); + if (coverage_blocks_long == NULL) { + + FATAL("Failed to g_hash_table_new, errno: %d", errno); + + } + } -static void instrument_coverage_write(GumAddress address, - GumStalkerOutput *output) { +bool instrument_write_inline(GumX86Writer *cw, GumAddress code_addr, + guint32 area_offset, guint32 area_offset_ror) { - afl_log_code code = {0}; - GumX86Writer *cw = output->writer.x86; - guint64 area_offset = instrument_get_offset_hash(address); - gsize map_size_pow2; - gsize area_offset_ror; - GumAddress code_addr = cw->pc; + afl_log_code code = {0}; code.code = template; - /* mov_prev_loc_curr_loc_shr1 */ - gssize prev_loc_value = GPOINTER_TO_SIZE(instrument_previous_pc_addr) - (code_addr + offsetof(afl_log_code, code.mov_prev_loc_curr_loc_shr1) + @@ -294,11 +424,7 @@ static void instrument_coverage_write(GumAddress address, offsetof(afl_log_code, code.mov_prev_loc_curr_loc_shr1) + sizeof(code.code.mov_prev_loc_curr_loc_shr1) - sizeof(gint) - sizeof(guint32); - if (!instrument_coverage_in_range(prev_loc_value)) { - - FATAL("Patch out of range (current_pc_value1): 0x%016lX", prev_loc_value); - - } + if (!instrument_coverage_in_range(prev_loc_value)) { return false; } *((gint *)&code.bytes[prev_loc_value_offset]) = (gint)prev_loc_value; @@ -311,11 +437,7 @@ static void instrument_coverage_write(GumAddress address, gssize prev_loc_value_offset2 = offsetof(afl_log_code, code.mov_eax_prev_loc) + sizeof(code.code.mov_eax_prev_loc) - sizeof(gint); - if (!instrument_coverage_in_range(prev_loc_value)) { - - FATAL("Patch out of range (current_pc_value1): 0x%016lX", prev_loc_value2); - - } + if (!instrument_coverage_in_range(prev_loc_value)) { return false; } *((gint *)&code.bytes[prev_loc_value_offset2]) = (gint)prev_loc_value2; @@ -338,12 +460,7 @@ static void instrument_coverage_write(GumAddress address, (code_addr + offsetof(afl_log_code, code.lea_rbx_area_ptr) + sizeof(code.code.lea_rbx_area_ptr))); - if (!instrument_coverage_in_range(lea_rbx_area_ptr_value)) { - - FATAL("Patch out of range (lea_rbx_area_ptr_value): 0x%016lX", - lea_rbx_area_ptr_value); - - } + if (!instrument_coverage_in_range(lea_rbx_area_ptr_value)) { return false; } *((guint32 *)&code.bytes[lea_rbx_area_ptr_offset]) = lea_rbx_area_ptr_value; @@ -353,12 +470,100 @@ static void instrument_coverage_write(GumAddress address, offsetof(afl_log_code, code.mov_prev_loc_curr_loc_shr1) + sizeof(code.code.mov_prev_loc_curr_loc_shr1) - sizeof(guint32); - map_size_pow2 = util_log2(__afl_map_size); - area_offset_ror = util_rotate(area_offset, 1, map_size_pow2); - *((guint32 *)&code.bytes[curr_loc_shr_1_offset]) = (guint32)(area_offset_ror); + if (instrument_suppress) { + + if (!g_hash_table_add(coverage_blocks, GSIZE_TO_POINTER(cw->code))) { + + FATAL("Failed - g_hash_table_add"); + + } + + } + gum_x86_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code)); + return true; + +} + +bool instrument_write_inline_long(GumX86Writer *cw, guint32 area_offset, + guint32 area_offset_ror) { + + afl_log_code_long code = {0}; + code.code = template_long; + + /* mov_rax_prev_loc_ptr1 */ + gssize mov_rax_prev_loc_ptr1_offset = + offsetof(afl_log_code_long, code.mov_rax_prev_loc_ptr1) + + sizeof(code.code.mov_rax_prev_loc_ptr1) - sizeof(gsize); + *((gsize *)&code.bytes[mov_rax_prev_loc_ptr1_offset]) = + GPOINTER_TO_SIZE(instrument_previous_pc_addr); + + /* xor_eax_curr_loc */ + gssize xor_eax_curr_loc_offset = + offsetof(afl_log_code_long, code.xor_eax_curr_loc) + + sizeof(code.code.xor_eax_curr_loc) - sizeof(guint32); + *((guint32 *)&code.bytes[xor_eax_curr_loc_offset]) = area_offset; + + /* mov_rbx_map_ptr */ + gsize mov_rbx_map_ptr_offset = + offsetof(afl_log_code_long, code.mov_rbx_map_ptr) + + sizeof(code.code.mov_rbx_map_ptr) - sizeof(gsize); + *((gsize *)&code.bytes[mov_rbx_map_ptr_offset]) = + GPOINTER_TO_SIZE(__afl_area_ptr); + + /* mov_rax_prev_loc_ptr2 */ + gssize mov_rax_prev_loc_ptr2_offset = + offsetof(afl_log_code_long, code.mov_rax_prev_loc_ptr2) + + sizeof(code.code.mov_rax_prev_loc_ptr2) - sizeof(gsize); + *((gsize *)&code.bytes[mov_rax_prev_loc_ptr2_offset]) = + GPOINTER_TO_SIZE(instrument_previous_pc_addr); + + /* mov_prev_loc_curr_loc_shr1 */ + gssize mov_prev_loc_curr_loc_shr1_offset = + offsetof(afl_log_code_long, code.mov_prev_loc_curr_loc_shr1) + + sizeof(code.code.mov_prev_loc_curr_loc_shr1) - sizeof(guint32); + *((guint32 *)&code.bytes[mov_prev_loc_curr_loc_shr1_offset]) = + (guint32)(area_offset_ror); + + if (instrument_suppress) { + + if (!g_hash_table_add(coverage_blocks_long, GSIZE_TO_POINTER(cw->code))) { + + FATAL("Failed - g_hash_table_add"); + + } + + } + + gum_x86_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code_long)); + return true; + +} + +static void instrument_coverage_write(GumAddress address, + GumStalkerOutput *output) { + + GumX86Writer *cw = output->writer.x86; + guint64 area_offset = (guint32)instrument_get_offset_hash(address); + gsize map_size_pow2; + guint32 area_offset_ror; + GumAddress code_addr = cw->pc; + + map_size_pow2 = util_log2(__afl_map_size); + area_offset_ror = (guint32)util_rotate(instrument_get_offset_hash(address), 1, + map_size_pow2); + + if (!instrument_write_inline(cw, code_addr, area_offset, area_offset_ror)) { + + if (!instrument_write_inline_long(cw, area_offset, area_offset_ror)) { + + FATAL("Failed to write inline instrumentation"); + + } + + } } @@ -380,17 +585,7 @@ void instrument_coverage_optimize(const cs_insn *instr, } - if (instrument_suppress) { - - instrument_coverage_suppress_init(); - - if (!g_hash_table_add(coverage_blocks, GSIZE_TO_POINTER(cw->code))) { - - FATAL("Failed - g_hash_table_add"); - - } - - } + if (instrument_suppress) { instrument_coverage_suppress_init(); } instrument_coverage_write(GUM_ADDRESS(instr->address), output); diff --git a/frida_mode/src/lib/lib.c b/frida_mode/src/lib/lib.c index d563b69b..7fac755a 100644 --- a/frida_mode/src/lib/lib.c +++ b/frida_mode/src/lib/lib.c @@ -44,8 +44,10 @@ static gboolean lib_find_exe(const GumModuleDetails *details, lib_details_t *lib_details = (lib_details_t *)user_data; - memcpy(lib_details->name, details->name, PATH_MAX); - memcpy(lib_details->path, details->path, PATH_MAX); + strncpy(lib_details->name, details->name, PATH_MAX); + strncpy(lib_details->path, details->path, PATH_MAX); + lib_details->name[PATH_MAX] = '\0'; + lib_details->path[PATH_MAX] = '\0'; lib_details->base_address = details->range->base_address; lib_details->size = details->range->size; return FALSE; diff --git a/frida_mode/src/lib/lib_apple.c b/frida_mode/src/lib/lib_apple.c index 634e0e30..d29d0303 100644 --- a/frida_mode/src/lib/lib_apple.c +++ b/frida_mode/src/lib/lib_apple.c @@ -17,8 +17,8 @@ static gboolean lib_get_main_module(const GumModuleDetails *details, GumDarwinModule **ret = (GumDarwinModule **)user_data; GumDarwinModule *module = gum_darwin_module_new_from_memory( - details->path, mach_task_self(), details->range->base_address, - GUM_DARWIN_MODULE_FLAGS_NONE, NULL); + details->path, mach_task_self(), details->range->base_address, + GUM_DARWIN_MODULE_FLAGS_NONE, NULL); FVERBOSE("Found main module: %s", module->name); diff --git a/frida_mode/src/main.c b/frida_mode/src/main.c index c8c50b37..9daf067b 100644 --- a/frida_mode/src/main.c +++ b/frida_mode/src/main.c @@ -49,10 +49,10 @@ extern void __libc_init(void *raw_args, void (*onexit)(void) __unused, int (*slingshot)(int, char **, char **), structors_array_t const *const structors); #else -extern int __libc_start_main(int (*main)(int, char **, char **), int argc, - char **ubp_av, void (*init)(void), - void (*fini)(void), void (*rtld_fini)(void), - void(*stack_end)); +extern int __libc_start_main(int (*main)(int, char **, char **), int argc, + char **ubp_av, void (*init)(void), + void (*fini)(void), void (*rtld_fini)(void), + void(*stack_end)); #endif typedef int (*main_fn_t)(int argc, char **argv, char **envp); @@ -166,7 +166,7 @@ static void afl_print_env(void) { if (fd < 0) { - FWARNF("Failed to open /proc/self/cmdline, errno: (%d)", errno); + FWARNF("Failed to open /proc/self/environ, errno: (%d)", errno); return; } @@ -174,7 +174,7 @@ static void afl_print_env(void) { ssize_t bytes_read = read(fd, buffer, PROC_MAX - 1); if (bytes_read < 0) { - FFATAL("Failed to read /proc/self/cmdline, errno: (%d)", errno); + FFATAL("Failed to read /proc/self/environ, errno: (%d)", errno); } @@ -197,7 +197,7 @@ static void afl_print_env(void) { } -__attribute__((visibility("default"))) void afl_frida_start(void) { +void afl_frida_config(void) { FOKF(cRED "**********************"); FOKF(cRED "* " cYEL "******************" cRED " *"); @@ -225,9 +225,7 @@ __attribute__((visibility("default"))) void afl_frida_start(void) { js_start(); - /* Initialize */ output_init(); - embedded_init(); entry_init(); instrument_init(); @@ -240,12 +238,35 @@ __attribute__((visibility("default"))) void afl_frida_start(void) { ranges_init(); stats_init(); - /* Start */ +} + +void afl_frida_run(void) { + stalker_start(); entry_start(); } +__attribute__((visibility("default"))) void afl_frida_start(void) { + + afl_frida_config(); + afl_frida_run(); + +} + +typedef void *(*entry_func_t)(size_t a1, size_t a2, size_t a3, size_t a4, + size_t a5, size_t a6); + +static void *on_entry(size_t a1, size_t a2, size_t a3, size_t a4, size_t a5, + size_t a6) { + + intercept_unhook(GSIZE_TO_POINTER(entry_point)); + afl_frida_run(); + entry_func_t entry = (entry_func_t)entry_point; + return entry(a1, a2, a3, a4, a5, a6); + +} + static int on_main(int argc, char **argv, char **envp) { int ret; @@ -254,7 +275,17 @@ static int on_main(int argc, char **argv, char **envp) { intercept_unhook_self(); - afl_frida_start(); + afl_frida_config(); + + if (entry_point == 0) { + + afl_frida_run(); + + } else { + + intercept_hook(GSIZE_TO_POINTER(entry_point), on_entry, NULL); + + } if (js_main_hook != NULL) { diff --git a/frida_mode/src/ranges.c b/frida_mode/src/ranges.c index e9fc3b4e..269ba59b 100644 --- a/frida_mode/src/ranges.c +++ b/frida_mode/src/ranges.c @@ -653,7 +653,7 @@ void ranges_init(void) { /* * After step 4 we have the total ranges to be instrumented, we now subtract * that either from the original ranges of the modules or from the whole - * memory if AFL_INST_NO_DYNAMIC_LOAD to configure the stalker. + * memory if AFL_FRIDA_INST_NO_DYNAMIC_LOAD to configure the stalker. */ if (ranges_inst_dynamic_load) { diff --git a/frida_mode/test/bloaty/GNUmakefile b/frida_mode/test/bloaty/GNUmakefile index 8e767fae..02a0a1e2 100644 --- a/frida_mode/test/bloaty/GNUmakefile +++ b/frida_mode/test/bloaty/GNUmakefile @@ -35,7 +35,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/cache/GNUmakefile b/frida_mode/test/cache/GNUmakefile index 12736a3f..98776193 100644 --- a/frida_mode/test/cache/GNUmakefile +++ b/frida_mode/test/cache/GNUmakefile @@ -11,7 +11,7 @@ QEMU_OUT:=$(BUILD_DIR)qemu-out FRIDA_OUT:=$(BUILD_DIR)frida-out ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFLPP_FRIDA_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/frida_hook.so diff --git a/frida_mode/test/cmov/GNUmakefile b/frida_mode/test/cmov/GNUmakefile index 96f1ae5b..0712e33b 100644 --- a/frida_mode/test/cmov/GNUmakefile +++ b/frida_mode/test/cmov/GNUmakefile @@ -11,7 +11,7 @@ QEMU_OUT:=$(BUILD_DIR)qemu-out FRIDA_OUT:=$(BUILD_DIR)frida-out ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFLPP_FRIDA_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/frida_hook.so diff --git a/frida_mode/test/cmplog/cmplog.c b/frida_mode/test/cmplog/cmplog.c index 2565b35c..d397f36e 100644 --- a/frida_mode/test/cmplog/cmplog.c +++ b/frida_mode/test/cmplog/cmplog.c @@ -2,7 +2,7 @@ // // Author: Mateusz Jurczyk (mjurczyk@google.com) // -// Copyright 2019-2023 Google LLC +// Copyright 2019-2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/frida_mode/test/deferred/GNUmakefile b/frida_mode/test/deferred/GNUmakefile index 22aeb2bf..e0b48797 100644 --- a/frida_mode/test/deferred/GNUmakefile +++ b/frida_mode/test/deferred/GNUmakefile @@ -10,7 +10,7 @@ TESTINSTSRC:=$(PWD)testinstr.c QEMU_OUT:=$(BUILD_DIR)qemu-out FRIDA_OUT:=$(BUILD_DIR)frida-out -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh ifndef ARCH diff --git a/frida_mode/test/deferred/testinstr.c b/frida_mode/test/deferred/testinstr.c index 0ab44582..4e5124ed 100644 --- a/frida_mode/test/deferred/testinstr.c +++ b/frida_mode/test/deferred/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/dynamic/GNUmakefile b/frida_mode/test/dynamic/GNUmakefile index f43416f7..6c577dff 100644 --- a/frida_mode/test/dynamic/GNUmakefile +++ b/frida_mode/test/dynamic/GNUmakefile @@ -17,7 +17,7 @@ FRIDA_OUT:=$(BUILD_DIR)frida-out AFLPP_FRIDA_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/frida_hook.so ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TESTINSTBIN) testinstr $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/dynamic/testinstr.c b/frida_mode/test/dynamic/testinstr.c index 8b285f6d..0abc61fd 100644 --- a/frida_mode/test/dynamic/testinstr.c +++ b/frida_mode/test/dynamic/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/entry_point/GNUmakefile b/frida_mode/test/entry_point/GNUmakefile index 08c660f7..b8c0ecb5 100644 --- a/frida_mode/test/entry_point/GNUmakefile +++ b/frida_mode/test/entry_point/GNUmakefile @@ -10,7 +10,7 @@ TESTINSTSRC:=$(PWD)testinstr.c QEMU_OUT:=$(BUILD_DIR)qemu-out FRIDA_OUT:=$(BUILD_DIR)frida-out -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh ifndef ARCH diff --git a/frida_mode/test/entry_point/testinstr.c b/frida_mode/test/entry_point/testinstr.c index 24d9a615..75e71bda 100644 --- a/frida_mode/test/entry_point/testinstr.c +++ b/frida_mode/test/entry_point/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/exe/testinstr.c b/frida_mode/test/exe/testinstr.c index d965502e..7b603659 100644 --- a/frida_mode/test/exe/testinstr.c +++ b/frida_mode/test/exe/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/freetype2/GNUmakefile b/frida_mode/test/freetype2/GNUmakefile index 8c35d5de..23318d52 100644 --- a/frida_mode/test/freetype2/GNUmakefile +++ b/frida_mode/test/freetype2/GNUmakefile @@ -64,7 +64,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/jpeg/GNUmakefile b/frida_mode/test/jpeg/GNUmakefile index a8242081..a4967039 100644 --- a/frida_mode/test/jpeg/GNUmakefile +++ b/frida_mode/test/jpeg/GNUmakefile @@ -47,7 +47,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/js/test.c b/frida_mode/test/js/test.c index 87c9cdf6..9799bf3b 100644 --- a/frida_mode/test/js/test.c +++ b/frida_mode/test/js/test.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/js/test2.c b/frida_mode/test/js/test2.c index 6b680a24..60b30eb5 100644 --- a/frida_mode/test/js/test2.c +++ b/frida_mode/test/js/test2.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/libpcap/GNUmakefile b/frida_mode/test/libpcap/GNUmakefile index 1bf9cd7f..745d7057 100644 --- a/frida_mode/test/libpcap/GNUmakefile +++ b/frida_mode/test/libpcap/GNUmakefile @@ -56,7 +56,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/libxml/GNUmakefile b/frida_mode/test/libxml/GNUmakefile index 6fc87585..f1f4a738 100644 --- a/frida_mode/test/libxml/GNUmakefile +++ b/frida_mode/test/libxml/GNUmakefile @@ -43,7 +43,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/libxslt/GNUmakefile b/frida_mode/test/libxslt/GNUmakefile index 655e652b..48bb0b40 100644 --- a/frida_mode/test/libxslt/GNUmakefile +++ b/frida_mode/test/libxslt/GNUmakefile @@ -42,7 +42,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/osx-lib/GNUmakefile b/frida_mode/test/osx-lib/GNUmakefile index 96dbb5ad..fdc9ec04 100644 --- a/frida_mode/test/osx-lib/GNUmakefile +++ b/frida_mode/test/osx-lib/GNUmakefile @@ -26,7 +26,7 @@ FRIDA_OUT:=$(BUILD_DIR)frida-out HARNESS_LDFLAGS:=-Wl,-no_pie LIB_CFLAGS:=-dynamiclib -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_MAIN_ADDR=$(shell $(GET_SYMBOL_ADDR) $(HARNESS_BIN) main 0x0) AFL_FRIDA_MAIN_ADDR2=$(shell $(GET_SYMBOL_ADDR) $(HARNESS2_BIN) main 0x0) AFL_FRIDA_FUZZ_ADDR=$(shell $(GET_SYMBOL_ADDR) $(HARNESS_BIN) LLVMFuzzerTestOneInput 0x0) diff --git a/frida_mode/test/output/testinstr.c b/frida_mode/test/output/testinstr.c index d965502e..7b603659 100644 --- a/frida_mode/test/output/testinstr.c +++ b/frida_mode/test/output/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/perf/GNUmakefile b/frida_mode/test/perf/GNUmakefile index 2d7c0239..6b49c2ba 100644 --- a/frida_mode/test/perf/GNUmakefile +++ b/frida_mode/test/perf/GNUmakefile @@ -31,7 +31,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/perf/perf.c b/frida_mode/test/perf/perf.c index d9626974..55efba26 100644 --- a/frida_mode/test/perf/perf.c +++ b/frida_mode/test/perf/perf.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/persistent_ret/GNUmakefile b/frida_mode/test/persistent_ret/GNUmakefile index 71f6a124..73d710a1 100644 --- a/frida_mode/test/persistent_ret/GNUmakefile +++ b/frida_mode/test/persistent_ret/GNUmakefile @@ -23,7 +23,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh ifeq "$(shell uname)" "Darwin" TEST_BIN_LDFLAGS:=-Wl,-no_pie diff --git a/frida_mode/test/persistent_ret/testinstr.c b/frida_mode/test/persistent_ret/testinstr.c index 12365ceb..85aa2b80 100644 --- a/frida_mode/test/persistent_ret/testinstr.c +++ b/frida_mode/test/persistent_ret/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/png/GNUmakefile b/frida_mode/test/png/GNUmakefile index 408b7dcb..eccc66f6 100644 --- a/frida_mode/test/png/GNUmakefile +++ b/frida_mode/test/png/GNUmakefile @@ -8,7 +8,7 @@ HARNESS_BUILD_DIR:=$(BUILD_DIR)harness/ PNGTEST_BUILD_DIR:=$(BUILD_DIR)pngtest/ LIBZ_FILE:=$(LIBZ_BUILD_DIR)zlib-1.2.13.tar.gz -LIBZ_URL:=http://www.zlib.net/zlib-1.2.13.tar.gz +LIBZ_URL:=http://www.zlib.net/fossils/zlib-1.2.13.tar.gz LIBZ_DIR:=$(LIBZ_BUILD_DIR)zlib-1.2.13/ LIBZ_PC:=$(LIBZ_DIR)zlib.pc LIBZ_LIB:=$(LIBZ_DIR)libz.a diff --git a/frida_mode/test/png/persistent/GNUmakefile b/frida_mode/test/png/persistent/GNUmakefile index 94e2be38..3dab713e 100644 --- a/frida_mode/test/png/persistent/GNUmakefile +++ b/frida_mode/test/png/persistent/GNUmakefile @@ -22,7 +22,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/png/persistent/hook/GNUmakefile b/frida_mode/test/png/persistent/hook/GNUmakefile index b6a1ca1a..f3d06c87 100644 --- a/frida_mode/test/png/persistent/hook/GNUmakefile +++ b/frida_mode/test/png/persistent/hook/GNUmakefile @@ -33,7 +33,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/proj4/GNUmakefile b/frida_mode/test/proj4/GNUmakefile index debc8a88..17850fa8 100644 --- a/frida_mode/test/proj4/GNUmakefile +++ b/frida_mode/test/proj4/GNUmakefile @@ -47,7 +47,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/re2/GNUmakefile b/frida_mode/test/re2/GNUmakefile index 220e7616..0b79210b 100644 --- a/frida_mode/test/re2/GNUmakefile +++ b/frida_mode/test/re2/GNUmakefile @@ -48,7 +48,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/sqlite/GNUmakefile b/frida_mode/test/sqlite/GNUmakefile index df470af8..6d3c7496 100644 --- a/frida_mode/test/sqlite/GNUmakefile +++ b/frida_mode/test/sqlite/GNUmakefile @@ -43,7 +43,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/testinstr/GNUmakefile b/frida_mode/test/testinstr/GNUmakefile index ebc0b2dc..46d9b85f 100644 --- a/frida_mode/test/testinstr/GNUmakefile +++ b/frida_mode/test/testinstr/GNUmakefile @@ -67,3 +67,8 @@ debug: --ex 'set environment LD_PRELOAD=$(ROOT)afl-frida-trace.so' \ --ex 'set disassembly-flavor intel' \ --args $(TESTINSTBIN) $(TESTINSTR_DATA_FILE) + +lldb: + lldb \ + -O 'settings set target.env-vars DYLD_INSERT_LIBRARIES=$(ROOT)afl-frida-trace.so' \ + -- $(TESTINSTBIN) $(TESTINSTR_DATA_FILE) diff --git a/frida_mode/test/testinstr/testinstr.c b/frida_mode/test/testinstr/testinstr.c index d965502e..7b603659 100644 --- a/frida_mode/test/testinstr/testinstr.c +++ b/frida_mode/test/testinstr/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/unstable/GNUmakefile b/frida_mode/test/unstable/GNUmakefile index 59b49449..3b7b6ddb 100644 --- a/frida_mode/test/unstable/GNUmakefile +++ b/frida_mode/test/unstable/GNUmakefile @@ -23,7 +23,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/test/unstable/unstable.c b/frida_mode/test/unstable/unstable.c index a87b6c74..16978e7e 100644 --- a/frida_mode/test/unstable/unstable.c +++ b/frida_mode/test/unstable/unstable.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/vorbis/GNUmakefile b/frida_mode/test/vorbis/GNUmakefile index 4cb5d417..b10d059e 100644 --- a/frida_mode/test/vorbis/GNUmakefile +++ b/frida_mode/test/vorbis/GNUmakefile @@ -54,7 +54,7 @@ endif endif ADDR_BIN:=$(ROOT)frida_mode/build/addr -GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/frida_get_symbol_addr.sh AFL_FRIDA_BASE_ADDR:=$(shell $(ADDR_BIN)) AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput $(AFL_FRIDA_BASE_ADDR)) diff --git a/frida_mode/update_frida_version.sh b/frida_mode/update_frida_version.sh index 18243fbb..2fafbf2f 100755 --- a/frida_mode/update_frida_version.sh +++ b/frida_mode/update_frida_version.sh @@ -2,7 +2,7 @@ test -n "$1" && { echo This script has no options. It updates the referenced Frida version in GNUmakefile to the most current one. ; exit 1 ; } OLD=$(grep -E '^GUM_DEVKIT_VERSION=' GNUmakefile 2>/dev/null|awk -F= '{print$2}') -NEW=$(curl https://github.com/frida/frida/releases/ 2>/dev/null|grep -E 'frida-gum-devkit-[0-9.]*-linux-x86_64'|head -n 1|sed 's/.*frida-gum-devkit-//'|sed 's/-linux.*//') +NEW=$(curl https://github.com/frida/frida/releases/ 2>/dev/null|grep 'Frida\ [0-9.]*'|head -n 1|sed 's/.*Frida\ //'| sed 's/<\/h2>//') echo Current set version: $OLD echo Newest available version: $NEW diff --git a/frida_mode/util/frida_get_symbol_addr.sh b/frida_mode/util/frida_get_symbol_addr.sh new file mode 100755 index 00000000..53d5b802 --- /dev/null +++ b/frida_mode/util/frida_get_symbol_addr.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright 2024 AFLplusplus +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +target="$1" +symbol="$2" +base="$3" + +test -z "$target" -o -z "$symbol" -o '!' -x "$target" && { + echo "Syntax: $0 executable function [baseaddress]" + echo + echo Help script to calculate the function address of a binary QEMU will load it to. + echo function is e.g. LLVMFuzzerTestOneInput, afl_qemu_driver_stdin, etc. + echo "baseaddress is tried to be auto-detected, you can use 'AFL_QEMU_DEBUG_MAPS=1 afl-qemu-trace ./executable' to see the maps." + exit 1 +} + +file=$(file $target|sed 's/.*: //') + +arch=$(echo $file|awk -F, '{print$2}'|tr -d ' ') +bits=$(echo $file|sed 's/-bit .*//'|sed 's/.* //') +pie=$(echo $file|grep -wqi pie && echo pie) +dso=$(echo $file|grep -wqi "shared object" && echo dso) + +test $(uname -s) = "Darwin" && symbol=_"$symbol" +tmp_addr=$(nm "$target" | grep -i "T $symbol" | awk '{print$1}' | tr a-f A-F) + +test -z "$tmp_addr" && { echo Error: function $symbol not found 1>&2; exit 1; } +test -z "$pie" && test -z "$dso" && { echo 0x$tmp_addr; exit 0; } + +test -z "$base" && { + test "$bits" = 32 -o "$bits" = 64 || { echo "Error: could not identify arch (bits=$bits)" 1>&2 ; exit 1; } + # is this true for arm/aarch64/i386 too? + base=0x555555554000 + #test "$arch" = Intel80386 && base=0x5555554000 + #test "$arch" = x86-64 && base=0x555555554000 + #test "$arch" = ARMaarch64 && base=0x5500000000 + # add more here, e.g. "$arch" = ARM +} + +test -z "$base" && { echo "Error: could not identify base address! bits=$bits arch=$arch" 1>&2 ; exit 1; } + +hex_base=$(echo "$base" | awk '{sub("^0x","");print $0}' | tr a-f A-F ) +echo $tmp_addr | echo "ibase=16;obase=10;$hex_base + $tmp_addr" | bc | tr A-F a-f | awk '{print "0x"$0}' +exit 0 diff --git a/frida_mode/util/get_symbol_addr.sh b/frida_mode/util/get_symbol_addr.sh deleted file mode 100755 index f5d8df91..00000000 --- a/frida_mode/util/get_symbol_addr.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# set -x -target="$1" -symbol="$2" -base="$3" - -test -z "$target" -o -z "$symbol" -o '!' -e "$target" && exit 0 - -test $(uname -s) = "Darwin" && symbol=_"$symbol" - -file "$target" | grep -q executable && { - nm "$target" | grep -i "T $symbol" | awk '{print"0x"$1}' - exit 0 -} - -hex_base=$(echo "$3" | awk '{sub("^0x","");print $0}' | tr a-f A-F ) -nm "$target" | grep -i "T $symbol" | awk '{print$1}' | tr a-f A-F | \ - xargs echo "ibase=16;obase=10;$hex_base + " | bc | tr A-F a-f | awk '{print "0x"$0}' -exit 0 diff --git a/include/afl-as.h b/include/afl-as.h index 486314e2..c005d43d 100644 --- a/include/afl-as.h +++ b/include/afl-as.h @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 8fb7ecb1..5efe5144 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, - Andrea Fioraldi <andreafioraldi@gmail.com>, - Dominik Maier <mail@dmnk.co> + Dominik Maier <mail@dmnk.co>, + Andrea Fioraldi <andreafioraldi@gmail.com>, and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ #define MESSAGES_TO_STDOUT #ifndef _GNU_SOURCE - #define _GNU_SOURCE 1 + #define _GNU_SOURCE #endif #ifndef _FILE_OFFSET_BITS #define _FILE_OFFSET_BITS 64 @@ -124,6 +124,10 @@ #define CASE_PREFIX "id_" #endif /* ^!SIMPLE_FILES */ +#ifdef AFL_PERSISTENT_RECORD + #define RECORD_PREFIX "RECORD:" +#endif + #define STAGE_BUF_SIZE (64) /* usable size for stage name buf in afl_state */ // Little helper to access the ptr to afl->##name_buf - for use in afl_realloc. @@ -149,6 +153,48 @@ struct tainted { }; +struct inf_profile { + + u32 inf_skipped_bytes; /* Inference Stage Profiling */ + u64 inf_execs_cost, inf_time_cost; + +}; + +/* ToDo: add cmplog profile as well */ +struct havoc_profile { + + u32 queued_det_stage, /* Det/Havoc Stage Profiling */ + queued_havoc_stage, total_queued_det, edge_det_stage, edge_havoc_stage, + total_det_edge; + + u64 det_stage_time, havoc_stage_time, total_det_time; + +}; + +struct skipdet_entry { + + u8 continue_inf, done_eff; + u32 undet_bits, quick_eff_bytes; + + u8 *skip_eff_map, /* we'v finish the eff_map */ + *done_inf_map; /* some bytes are not done yet */ + +}; + +struct skipdet_global { + + u8 use_skip_havoc; + + u32 undet_bits_threshold; + + u64 last_cov_undet; + + u8 *virgin_det_bits; /* global fuzzed bits */ + + struct inf_profile *inf_prof; + +}; + struct queue_entry { u8 *fname; /* File name for the test case */ @@ -157,6 +203,7 @@ struct queue_entry { u8 colorized, /* Do not run redqueen stage again */ cal_failed; /* Calibration failed? */ + bool trim_done, /* Trimmed? */ was_fuzzed, /* historical, but needed for MOpt */ passed_det, /* Deterministic stages passed? */ @@ -168,22 +215,21 @@ struct queue_entry { disabled; /* Is disabled from fuzz selection */ u32 bitmap_size, /* Number of bits set in bitmap */ - fuzz_level, /* Number of fuzzing iterations */ - n_fuzz_entry /* offset in n_fuzz */ #ifdef INTROSPECTION - , stats_selected, /* stats: how often selected */ stats_skipped, /* stats: how often skipped */ stats_finds, /* stats: # of saved finds */ stats_crashes, /* stats: # of saved crashes */ - stats_tmouts /* stats: # of saved timeouts */ + stats_tmouts, /* stats: # of saved timeouts */ #endif - ; + fuzz_level, /* Number of fuzzing iterations */ + n_fuzz_entry; /* offset in n_fuzz */ u64 exec_us, /* Execution time (us) */ handicap, /* Number of queue cycles behind */ depth, /* Path depth */ exec_cksum, /* Checksum of the execution trace */ + custom, /* Marker for custom mutators */ stats_mutated; /* stats: # of mutations performed */ u8 *trace_mini; /* Trace bytes, if kept */ @@ -203,6 +249,8 @@ struct queue_entry { struct queue_entry *mother; /* queue entry this based on */ + struct skipdet_entry *skipdet_e; + }; struct extra_data { @@ -247,6 +295,8 @@ enum { /* 19 */ STAGE_CUSTOM_MUTATOR, /* 20 */ STAGE_COLORIZATION, /* 21 */ STAGE_ITS, + /* 22 */ STAGE_INF, + /* 23 */ STAGE_QUICK, STAGE_NUM_MAX @@ -345,6 +395,7 @@ enum { /* 13 */ PY_FUNC_DESCRIBE, /* 14 */ PY_FUNC_FUZZ_SEND, /* 15 */ PY_FUNC_SPLICE_OPTOUT, + /* 16 */ PY_FUNC_POST_RUN, PY_FUNC_COUNT }; @@ -400,7 +451,9 @@ typedef struct afl_env_vars { afl_exit_on_seed_issues, afl_try_affinity, afl_ignore_problems, afl_keep_timeouts, afl_no_crash_readme, afl_ignore_timeouts, afl_no_startup_calibration, afl_no_warn_instability, - afl_post_process_keep_original; + afl_post_process_keep_original, afl_crashing_seeds_as_new_crash, + afl_final_sync, afl_ignore_seed_problems, afl_disable_redundant, + afl_sha1_filenames; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_preload, @@ -493,7 +546,8 @@ typedef struct afl_state { *orig_cmdline, /* Original command line */ *infoexec; /* Command to execute on a new crash */ - u32 hang_tmout; /* Timeout used for hang det (ms) */ + u32 hang_tmout, /* Timeout used for hang det (ms) */ + stats_update_freq; /* Stats update frequency (execs) */ u8 havoc_stack_pow2, /* HAVOC_STACK_POW2 */ no_unlink, /* do not unlink cur_input */ @@ -502,14 +556,12 @@ typedef struct afl_state { custom_splice_optout, /* Custom mutator no splice buffer */ is_main_node, /* if this is the main node */ is_secondary_node, /* if this is a secondary instance */ - pizza_is_served; /* pizza mode */ - - u32 stats_update_freq; /* Stats update frequency (execs) */ - - u8 schedule; /* Power schedule (default: EXPLORE)*/ - u8 havoc_max_mult; - - u8 skip_deterministic, /* Skip deterministic stages? */ + pizza_is_served, /* pizza mode */ + input_mode, /* target wants text inputs */ + fuzz_mode, /* coverage/exploration or crash/exploitation mode */ + schedule, /* Power schedule (default: EXPLORE)*/ + havoc_max_mult, /* havoc multiplier */ + skip_deterministic, /* Skip deterministic stages? */ use_splicing, /* Recombine input files? */ non_instrumented_mode, /* Run in non-instrumented mode? */ score_changed, /* Scoring for favorites changed? */ @@ -596,7 +648,11 @@ typedef struct afl_state { last_hang_time, /* Time for most recent hang (ms) */ longest_find_time, /* Longest time taken for a find */ exit_on_time, /* Delay to exit if no new paths */ - sync_time; /* Sync time (ms) */ + sync_time, /* Sync time (ms) */ + switch_fuzz_mode, /* auto or fixed fuzz mode */ + calibration_time_us, /* Time spend on calibration */ + sync_time_us, /* Time spend on sync */ + trim_time_us; /* Time spend on trimming */ u32 slowest_exec_ms, /* Slowest testcase non hang in ms */ subseq_tmouts; /* Number of timeouts in a row */ @@ -609,6 +665,7 @@ typedef struct afl_state { u32 stage_cur, stage_max; /* Stage progression */ s32 splicing_with; /* Splicing with which test case? */ + s64 smallest_favored; /* smallest queue id favored */ u32 main_node_id, main_node_max; /* Main instance job splitting */ @@ -673,7 +730,8 @@ typedef struct afl_state { u32 cmplog_max_filesize; u32 cmplog_lvl; u32 colorize_success; - u8 cmplog_enable_arith, cmplog_enable_transform, cmplog_random_colorization; + u8 cmplog_enable_arith, cmplog_enable_transform, cmplog_enable_scale, + cmplog_enable_xtreme_transform, cmplog_random_colorization; struct afl_pass_stat *pass_stats; struct cmp_map *orig_cmp_map; @@ -778,6 +836,11 @@ typedef struct afl_state { * is too large) */ struct queue_entry **q_testcase_cache; + /* Global Profile Data for deterministic/havoc-splice stage */ + struct havoc_profile *havoc_prof; + + struct skipdet_global *skipdet_g; + #ifdef INTROSPECTION char mutation[8072]; char m_tmp[4096]; @@ -1018,6 +1081,16 @@ struct custom_mutator { void (*afl_custom_fuzz_send)(void *data, const u8 *buf, size_t buf_size); /** + * This method can be used if you want to run some code or scripts each time + * AFL++ executes the target with afl-fuzz. + * + * (Optional) + * + * @param data pointer returned in afl_custom_init by this custom mutator + */ + void (*afl_custom_post_run)(void *data); + + /** * Allow for additional analysis (e.g. calling a different tool that does a * different kind of coverage and saves this for the custom mutator). * @@ -1072,6 +1145,7 @@ void finalize_py_module(void *); u32 fuzz_count_py(void *, const u8 *, size_t); void fuzz_send_py(void *, const u8 *, size_t); +void post_run_py(void *); size_t post_process_py(void *, u8 *, size_t, u8 **); s32 init_trim_py(void *, u8 *, size_t); s32 post_trim_py(void *, u8); @@ -1145,6 +1219,10 @@ void show_stats_normal(afl_state_t *); void show_stats_pizza(afl_state_t *); void show_init_stats(afl_state_t *); +void update_calibration_time(afl_state_t *afl, u64 *time); +void update_trim_time(afl_state_t *afl, u64 *time); +void update_sync_time(afl_state_t *afl, u64 *time); + /* StatsD */ void statsd_setup_format(afl_state_t *afl); @@ -1202,6 +1280,7 @@ u8 check_if_text_buf(u8 *buf, u32 len); #ifndef AFL_SHOWMAP void setup_signal_handlers(void); #endif +char *get_fuzzing_state(afl_state_t *afl); /* CmpLog */ @@ -1216,6 +1295,13 @@ AFL_RAND_RETURN rand_next(afl_state_t *afl); /* probability between 0.0 and 1.0 */ double rand_next_percent(afl_state_t *afl); +/* SkipDet Functions */ + +u8 skip_deterministic_stage(afl_state_t *, u8 *, u8 *, u32, u64); +u8 is_det_timeout(u64, u8); + +void plot_profile_data(afl_state_t *, struct queue_entry *); + /**** Inline routines ****/ /* Generate a random number (from 0 to limit - 1). This may @@ -1319,6 +1405,32 @@ void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q, u8 *in, void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q, u8 *mem); +/* Compute the SHA1 hash of `data`, which is of `len` bytes, and return the + * result as a `\0`-terminated hex string, which the caller much `ck_free`. */ +char *sha1_hex(const u8 *data, size_t len); + +/* Apply `sha1_hex` to the first `len` bytes of data of the file at `fname`. */ +char *sha1_hex_for_file(const char *fname, u32 len); + +/* Create file `fn`, but allow it to already exist if `AFL_SHA1_FILENAMES` is + * enabled. */ +static inline int permissive_create(afl_state_t *afl, const char *fn) { + + int fd = open(fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); + if (unlikely(fd < 0)) { + + if (!(afl->afl_env.afl_sha1_filenames && errno == EEXIST)) { + + PFATAL("Unable to create '%s'", fn); + + } + + } + + return fd; + +} + #if TESTCASE_CACHE == 1 #error define of TESTCASE_CACHE must be zero or larger than 1 #endif diff --git a/include/afl-mutations.h b/include/afl-mutations.h new file mode 100644 index 00000000..79cf7c6a --- /dev/null +++ b/include/afl-mutations.h @@ -0,0 +1,2681 @@ +/* Implementation of afl havoc mutation to be used in AFL++ custom mutators and + partially in afl-fuzz itself. + + How to use: + + #include "afl-mutations.h" // needs afl-fuzz.h + + u32 afl_mutate(afl_state_t *afl, u8 *buf, u32 len, u32t steps, bool is_text, + bool is_exploration, u8 *splice_buf, u32 splice_len, + u32 max_len); + + Returns: + u32 - the length of the mutated data return in *buf. 0 = error + Parameters: + afl_state_t *afl - the *afl state pointer + u8 *buf - the input buffer to mutate which will be mutated into. + NOTE: must be able to contain a size of at least max_len!! (see below) + u32 len - the length of the input + u32 steps - how many mutations to perform on the input + bool is_text - is the target expecting text inputs + bool is_exploration - mutate for exploration mode (instead of exploitation) + splice_buf - a buffer from another corpus item to splice with. + If NULL then no splicing is done (obviously). + splice_len - the length of the splice buffer. If 0 then no splicing. + u32 max_len - the maximum size the mutated buffer may grow to +*/ + +#ifndef AFL_MUTATIONS_H +#define AFL_MUTATIONS_H + +#include <stdbool.h> +#include <inttypes.h> + +#define MUT_STRATEGY_ARRAY_SIZE 256 + +s8 interesting_8[] = {INTERESTING_8}; +s16 interesting_16[] = {INTERESTING_8, INTERESTING_16}; +s32 interesting_32[] = {INTERESTING_8, INTERESTING_16, INTERESTING_32}; + +enum { + + /* 00 */ MUT_FLIPBIT, + /* 01 */ MUT_INTERESTING8, + /* 02 */ MUT_INTERESTING16, + /* 03 */ MUT_INTERESTING16BE, + /* 04 */ MUT_INTERESTING32, + /* 05 */ MUT_INTERESTING32BE, + /* 06 */ MUT_ARITH8_, + /* 07 */ MUT_ARITH8, + /* 08 */ MUT_ARITH16_, + /* 09 */ MUT_ARITH16BE_, + /* 10 */ MUT_ARITH16, + /* 11 */ MUT_ARITH16BE, + /* 12 */ MUT_ARITH32_, + /* 13 */ MUT_ARITH32BE_, + /* 14 */ MUT_ARITH32, + /* 15 */ MUT_ARITH32BE, + /* 16 */ MUT_RAND8, + /* 17 */ MUT_CLONE_COPY, + /* 18 */ MUT_CLONE_FIXED, + /* 19 */ MUT_OVERWRITE_COPY, + /* 20 */ MUT_OVERWRITE_FIXED, + /* 21 */ MUT_BYTEADD, + /* 22 */ MUT_BYTESUB, + /* 23 */ MUT_FLIP8, + /* 24 */ MUT_SWITCH, + /* 25 */ MUT_DEL, + /* 26 */ MUT_SHUFFLE, + /* 27 */ MUT_DELONE, + /* 28 */ MUT_INSERTONE, + /* 29 */ MUT_ASCIINUM, + /* 30 */ MUT_INSERTASCIINUM, + /* 31 */ MUT_EXTRA_OVERWRITE, + /* 32 */ MUT_EXTRA_INSERT, + /* 33 */ MUT_AUTO_EXTRA_OVERWRITE, + /* 34 */ MUT_AUTO_EXTRA_INSERT, + /* 35 */ MUT_SPLICE_OVERWRITE, + /* 36 */ MUT_SPLICE_INSERT, + + MUT_MAX + +}; + +#define MUT_TXT_ARRAY_SIZE 200 +u32 text_array[MUT_TXT_ARRAY_SIZE] = {MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT}; + +#define MUT_BIN_ARRAY_SIZE 256 +u32 binary_array[MUT_BIN_ARRAY_SIZE] = {MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT}; + +#define MUT_NORMAL_ARRAY_SIZE 77 +u32 normal_splice_array[MUT_NORMAL_ARRAY_SIZE] = {MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_FIXED, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_FIXED, + MUT_BYTEADD, + MUT_BYTESUB, + MUT_FLIP8, + MUT_SWITCH, + MUT_SWITCH, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT}; + +#define MUT_SPLICE_ARRAY_SIZE 81 +u32 full_splice_array[MUT_SPLICE_ARRAY_SIZE] = {MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_FIXED, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_FIXED, + MUT_BYTEADD, + MUT_BYTESUB, + MUT_FLIP8, + MUT_SWITCH, + MUT_SWITCH, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT}; + +u32 mutation_strategy_exploration_text[MUT_STRATEGY_ARRAY_SIZE] = { + + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT + +}; + +u32 mutation_strategy_exploration_binary[MUT_STRATEGY_ARRAY_SIZE] = { + + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT + +}; + +u32 mutation_strategy_exploitation_text[MUT_STRATEGY_ARRAY_SIZE] = { + + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT + +}; + +u32 mutation_strategy_exploitation_binary[MUT_STRATEGY_ARRAY_SIZE] = { + + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_FLIPBIT, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING8, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING16BE, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_INTERESTING32BE, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8_, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH8, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16BE_, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH16BE, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32BE_, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_ARITH32BE, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_RAND8, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_COPY, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_CLONE_FIXED, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_COPY, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_OVERWRITE_FIXED, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTEADD, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_BYTESUB, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_FLIP8, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_SWITCH, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_DEL, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_SHUFFLE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_DELONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_INSERTONE, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_ASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_INSERTASCIINUM, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_OVERWRITE, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_EXTRA_INSERT, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_OVERWRITE, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_AUTO_EXTRA_INSERT, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_OVERWRITE, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT, + MUT_SPLICE_INSERT + +}; + +u32 afl_mutate(afl_state_t *, u8 *, u32, u32, bool, bool, u8 *, u32, u32); +u32 choose_block_len(afl_state_t *, u32); + +/* Helper to choose random block len for block operations in fuzz_one(). + Doesn't return zero, provided that max_len is > 0. */ + +inline u32 choose_block_len(afl_state_t *afl, u32 limit) { + + u32 min_value, max_value; + u32 rlim = MIN(afl->queue_cycle, (u32)3); + + if (unlikely(!afl->run_over10m)) { rlim = 1; } + + switch (rand_below(afl, rlim)) { + + case 0: + min_value = 1; + max_value = HAVOC_BLK_SMALL; + break; + + case 1: + min_value = HAVOC_BLK_SMALL; + max_value = HAVOC_BLK_MEDIUM; + break; + + default: + + if (likely(rand_below(afl, 10))) { + + min_value = HAVOC_BLK_MEDIUM; + max_value = HAVOC_BLK_LARGE; + + } else { + + min_value = HAVOC_BLK_LARGE; + max_value = HAVOC_BLK_XL; + + } + + } + + if (min_value >= limit) { min_value = 1; } + + return min_value + rand_below(afl, MIN(max_value, limit) - min_value + 1); + +} + +inline u32 afl_mutate(afl_state_t *afl, u8 *buf, u32 len, u32 steps, + bool is_text, bool is_exploration, u8 *splice_buf, + u32 splice_len, u32 max_len) { + + if (!buf || !len) { return 0; } + + u32 *mutation_array; + static u8 *tmp_buf = NULL; + static u32 tmp_buf_size = 0; + + if (max_len > tmp_buf_size) { + + if (tmp_buf) { + + u8 *ptr = realloc(tmp_buf, max_len); + + if (!ptr) { + + return 0; + + } else { + + tmp_buf = ptr; + + } + + } else { + + if ((tmp_buf = malloc(max_len)) == NULL) { return 0; } + + } + + tmp_buf_size = max_len; + + } + + if (is_text) { + + if (is_exploration) { + + mutation_array = (u32 *)&mutation_strategy_exploration_text; + + } else { + + mutation_array = (u32 *)&mutation_strategy_exploitation_text; + + } + + } else { + + if (is_exploration) { + + mutation_array = (u32 *)&mutation_strategy_exploration_binary; + + } else { + + mutation_array = (u32 *)&mutation_strategy_exploitation_binary; + + } + + } + + for (u32 step = 0; step < steps; ++step) { + + retry_havoc_step: { + + u32 r = rand_below(afl, MUT_STRATEGY_ARRAY_SIZE), item; + + switch (mutation_array[r]) { + + case MUT_FLIPBIT: { + + /* Flip a single bit somewhere. Spooky! */ + u8 bit = rand_below(afl, 8); + u32 off = rand_below(afl, len); + buf[off] ^= 1 << bit; + + break; + + } + + case MUT_INTERESTING8: { + + /* Set byte to interesting value. */ + + item = rand_below(afl, sizeof(interesting_8)); + buf[rand_below(afl, len)] = interesting_8[item]; + break; + + } + + case MUT_INTERESTING16: { + + /* Set word to interesting value, little endian. */ + + if (unlikely(len < 2)) { break; } // no retry + + item = rand_below(afl, sizeof(interesting_16) >> 1); + *(u16 *)(buf + rand_below(afl, len - 1)) = interesting_16[item]; + + break; + + } + + case MUT_INTERESTING16BE: { + + /* Set word to interesting value, big endian. */ + + if (unlikely(len < 2)) { break; } // no retry + + item = rand_below(afl, sizeof(interesting_16) >> 1); + *(u16 *)(buf + rand_below(afl, len - 1)) = SWAP16(interesting_16[item]); + + break; + + } + + case MUT_INTERESTING32: { + + /* Set dword to interesting value, little endian. */ + + if (unlikely(len < 4)) { break; } // no retry + + item = rand_below(afl, sizeof(interesting_32) >> 2); + *(u32 *)(buf + rand_below(afl, len - 3)) = interesting_32[item]; + + break; + + } + + case MUT_INTERESTING32BE: { + + /* Set dword to interesting value, big endian. */ + + if (unlikely(len < 4)) { break; } // no retry + + item = rand_below(afl, sizeof(interesting_32) >> 2); + *(u32 *)(buf + rand_below(afl, len - 3)) = SWAP32(interesting_32[item]); + + break; + + } + + case MUT_ARITH8_: { + + /* Randomly subtract from byte. */ + + item = 1 + rand_below(afl, ARITH_MAX); + buf[rand_below(afl, len)] -= item; + break; + + } + + case MUT_ARITH8: { + + /* Randomly add to byte. */ + + item = 1 + rand_below(afl, ARITH_MAX); + buf[rand_below(afl, len)] += item; + break; + + } + + case MUT_ARITH16_: { + + /* Randomly subtract from word, little endian. */ + + if (unlikely(len < 2)) { break; } // no retry + + u32 pos = rand_below(afl, len - 1); + item = 1 + rand_below(afl, ARITH_MAX); + *(u16 *)(buf + pos) -= item; + + break; + + } + + case MUT_ARITH16BE_: { + + /* Randomly subtract from word, big endian. */ + + if (unlikely(len < 2)) { break; } // no retry + + u32 pos = rand_below(afl, len - 1); + u16 num = 1 + rand_below(afl, ARITH_MAX); + *(u16 *)(buf + pos) = SWAP16(SWAP16(*(u16 *)(buf + pos)) - num); + + break; + + } + + case MUT_ARITH16: { + + /* Randomly add to word, little endian. */ + + if (unlikely(len < 2)) { break; } // no retry + + u32 pos = rand_below(afl, len - 1); + item = 1 + rand_below(afl, ARITH_MAX); + *(u16 *)(buf + pos) += item; + + break; + + } + + case MUT_ARITH16BE: { + + /* Randomly add to word, big endian. */ + + if (unlikely(len < 2)) { break; } // no retry + + u32 pos = rand_below(afl, len - 1); + u16 num = 1 + rand_below(afl, ARITH_MAX); + *(u16 *)(buf + pos) = SWAP16(SWAP16(*(u16 *)(buf + pos)) + num); + + break; + + } + + case MUT_ARITH32_: { + + /* Randomly subtract from dword, little endian. */ + + if (unlikely(len < 4)) { break; } // no retry + + u32 pos = rand_below(afl, len - 3); + item = 1 + rand_below(afl, ARITH_MAX); + *(u32 *)(buf + pos) -= item; + + break; + + } + + case MUT_ARITH32BE_: { + + /* Randomly subtract from dword, big endian. */ + + if (unlikely(len < 4)) { break; } // no retry + + u32 pos = rand_below(afl, len - 3); + u32 num = 1 + rand_below(afl, ARITH_MAX); + *(u32 *)(buf + pos) = SWAP32(SWAP32(*(u32 *)(buf + pos)) - num); + + break; + + } + + case MUT_ARITH32: { + + /* Randomly add to dword, little endian. */ + + if (unlikely(len < 4)) { break; } // no retry + + u32 pos = rand_below(afl, len - 3); + item = 1 + rand_below(afl, ARITH_MAX); + *(u32 *)(buf + pos) += item; + + break; + + } + + case MUT_ARITH32BE: { + + /* Randomly add to dword, big endian. */ + + if (unlikely(len < 4)) { break; } // no retry + + u32 pos = rand_below(afl, len - 3); + u32 num = 1 + rand_below(afl, ARITH_MAX); + *(u32 *)(buf + pos) = SWAP32(SWAP32(*(u32 *)(buf + pos)) + num); + + break; + + } + + case MUT_RAND8: { + + /* Just set a random byte to a random value. Because, + why not. We use XOR with 1-255 to eliminate the + possibility of a no-op. */ + + u32 pos = rand_below(afl, len); + item = 1 + rand_below(afl, 255); + buf[pos] ^= item; + break; + + } + + case MUT_CLONE_COPY: { + + if (likely(len + HAVOC_BLK_XL < max_len)) { + + /* Clone bytes. */ + + u32 clone_len = choose_block_len(afl, len); + u32 clone_from = rand_below(afl, len - clone_len + 1); + u32 clone_to = rand_below(afl, len); + + /* Head */ + + memcpy(tmp_buf, buf, clone_to); + + /* Inserted part */ + + memcpy(tmp_buf + clone_to, buf + clone_from, clone_len); + + /* Tail */ + memcpy(tmp_buf + clone_to + clone_len, buf + clone_to, + len - clone_to); + + len += clone_len; + memcpy(buf, tmp_buf, len); + + } else if (unlikely(len < 8)) { + + break; + + } else { + + goto retry_havoc_step; + + } + + break; + + } + + case MUT_CLONE_FIXED: { + + if (likely(len + HAVOC_BLK_XL < max_len)) { + + /* Insert a block of constant bytes (25%). */ + + u32 clone_len = choose_block_len(afl, HAVOC_BLK_XL); + u32 clone_to = rand_below(afl, len); + u32 strat = rand_below(afl, 2); + u32 clone_from = clone_to ? clone_to - 1 : 0; + item = strat ? rand_below(afl, 256) : buf[clone_from]; + + /* Head */ + + memcpy(tmp_buf, buf, clone_to); + + /* Inserted part */ + + memset(tmp_buf + clone_to, item, clone_len); + + /* Tail */ + memcpy(tmp_buf + clone_to + clone_len, buf + clone_to, + len - clone_to); + + len += clone_len; + memcpy(buf, tmp_buf, len); + + } else if (unlikely(len < 8)) { + + break; + + } else { + + goto retry_havoc_step; + + } + + break; + + } + + case MUT_OVERWRITE_COPY: { + + /* Overwrite bytes with a randomly selected chunk bytes. */ + + if (unlikely(len < 2)) { break; } // no retry + + u32 copy_len = choose_block_len(afl, len - 1); + u32 copy_from = rand_below(afl, len - copy_len + 1); + u32 copy_to = rand_below(afl, len - copy_len + 1); + + if (likely(copy_from != copy_to)) { + + memmove(buf + copy_to, buf + copy_from, copy_len); + + } + + break; + + } + + case MUT_OVERWRITE_FIXED: { + + /* Overwrite bytes with fixed bytes. */ + + if (unlikely(len < 2)) { break; } // no retry + + u32 copy_len = choose_block_len(afl, len - 1); + u32 copy_to = rand_below(afl, len - copy_len + 1); + u32 strat = rand_below(afl, 2); + u32 copy_from = copy_to ? copy_to - 1 : 0; + item = strat ? rand_below(afl, 256) : buf[copy_from]; + memset(buf + copy_to, item, copy_len); + + break; + + } + + case MUT_BYTEADD: { + + /* Increase byte by 1. */ + + buf[rand_below(afl, len)]++; + break; + + } + + case MUT_BYTESUB: { + + /* Decrease byte by 1. */ + + buf[rand_below(afl, len)]--; + break; + + } + + case MUT_FLIP8: { + + /* Flip byte. */ + + buf[rand_below(afl, len)] ^= 0xff; + break; + + } + + case MUT_SWITCH: { + + if (unlikely(len < 4)) { break; } // no retry + + /* Switch bytes. */ + + u32 to_end, switch_to, switch_len, switch_from; + switch_from = rand_below(afl, len); + do { + + switch_to = rand_below(afl, len); + + } while (unlikely(switch_from == switch_to)); + + if (switch_from < switch_to) { + + switch_len = switch_to - switch_from; + to_end = len - switch_to; + + } else { + + switch_len = switch_from - switch_to; + to_end = len - switch_from; + + } + + switch_len = choose_block_len(afl, MIN(switch_len, to_end)); + + /* Backup */ + + memcpy(tmp_buf, buf + switch_from, switch_len); + + /* Switch 1 */ + + memcpy(buf + switch_from, buf + switch_to, switch_len); + + /* Switch 2 */ + + memcpy(buf + switch_to, tmp_buf, switch_len); + + break; + + } + + case MUT_DEL: { + + /* Delete bytes. */ + + if (unlikely(len < 2)) { break; } // no retry + + /* Don't delete too much. */ + + u32 del_len = choose_block_len(afl, len - 1); + u32 del_from = rand_below(afl, len - del_len + 1); + memmove(buf + del_from, buf + del_from + del_len, + len - del_from - del_len); + len -= del_len; + + break; + + } + + case MUT_SHUFFLE: { + + /* Shuffle bytes. */ + + if (unlikely(len < 4)) { break; } // no retry + + u32 blen = choose_block_len(afl, len - 1); + u32 off = rand_below(afl, len - blen + 1); + + for (u32 i = blen - 1; i > 0; i--) { + + u32 j; + do { + + j = rand_below(afl, i + 1); + + } while (unlikely(i == j)); + + u8 temp = buf[off + i]; + buf[off + i] = buf[off + j]; + buf[off + j] = temp; + + } + + break; + + } + + case MUT_DELONE: { + + /* Delete bytes. */ + + if (unlikely(len < 2)) { break; } // no retry + + /* Don't delete too much. */ + + u32 del_len = 1; + u32 del_from = rand_below(afl, len - del_len + 1); + memmove(buf + del_from, buf + del_from + del_len, + len - del_from - del_len); + + len -= del_len; + + break; + + } + + case MUT_INSERTONE: { + + if (unlikely(len < 2)) { break; } // no retry + + u32 clone_len = 1; + u32 clone_to = rand_below(afl, len); + u32 strat = rand_below(afl, 2); + u32 clone_from = clone_to ? clone_to - 1 : 0; + item = strat ? rand_below(afl, 256) : buf[clone_from]; + + /* Head */ + + memcpy(tmp_buf, buf, clone_to); + + /* Inserted part */ + + memset(tmp_buf + clone_to, item, clone_len); + + /* Tail */ + memcpy(tmp_buf + clone_to + clone_len, buf + clone_to, len - clone_to); + + len += clone_len; + memcpy(buf, tmp_buf, len); + + break; + + } + + case MUT_ASCIINUM: { + + if (unlikely(len < 4)) { break; } // no retry + + u32 off = rand_below(afl, len), off2 = off, cnt = 0; + + while (off2 + cnt < len && !isdigit(buf[off2 + cnt])) { + + ++cnt; + + } + + // none found, wrap + if (off2 + cnt == len) { + + off2 = 0; + cnt = 0; + + while (cnt < off && !isdigit(buf[off2 + cnt])) { + + ++cnt; + + } + + if (cnt == off) { + + if (len < 8) { + + break; + + } else { + + goto retry_havoc_step; + + } + + } + + } + + off = off2 + cnt; + off2 = off + 1; + + while (off2 < len && isdigit(buf[off2])) { + + ++off2; + + } + + s64 val = buf[off] - '0'; + for (u32 i = off + 1; i < off2; ++i) { + + val = (val * 10) + buf[i] - '0'; + + } + + if (off && buf[off - 1] == '-') { val = -val; } + + u32 strat = rand_below(afl, 8); + switch (strat) { + + case 0: + val++; + break; + case 1: + val--; + break; + case 2: + val *= 2; + break; + case 3: + val /= 2; + break; + case 4: + if (likely(val && (u64)val < 0x19999999)) { + + val = (u64)rand_next(afl) % (u64)((u64)val * 10); + + } else { + + val = rand_below(afl, 256); + + } + + break; + case 5: + val += rand_below(afl, 256); + break; + case 6: + val -= rand_below(afl, 256); + break; + case 7: + val = ~(val); + break; + + } + + char numbuf[32]; + snprintf(numbuf, sizeof(buf), "%" PRId64, val); + u32 old_len = off2 - off; + u32 new_len = strlen(numbuf); + + if (old_len == new_len) { + + memcpy(buf + off, numbuf, new_len); + + } else { + + /* Head */ + + memcpy(tmp_buf, buf, off); + + /* Inserted part */ + + memcpy(tmp_buf + off, numbuf, new_len); + + /* Tail */ + memcpy(tmp_buf + off + new_len, buf + off2, len - off2); + + len += (new_len - old_len); + memcpy(buf, tmp_buf, len); + + } + + // fprintf(stderr, "AFTER : %s\n", buf); + break; + + } + + case MUT_INSERTASCIINUM: { + + u32 ins_len = 1 + rand_below(afl, 8); + u32 pos = rand_below(afl, len); + + /* Insert ascii number. */ + if (unlikely(len < pos + ins_len)) { + + // no retry if we have a small input + if (unlikely(len < 8)) { + + break; + + } else { + + goto retry_havoc_step; + + } + + } + + u64 val = rand_next(afl); + char numbuf[32]; + snprintf(numbuf, sizeof(numbuf), "%llu", val); + size_t val_len = strlen(numbuf), off; + + if (ins_len > val_len) { + + ins_len = val_len; + off = 0; + + } else { + + off = val_len - ins_len; + + } + + memcpy(buf + pos, numbuf + off, ins_len); + + break; + + } + + case MUT_EXTRA_OVERWRITE: { + + if (unlikely(!afl->extras_cnt)) { goto retry_havoc_step; } + + /* Use the dictionary. */ + + u32 use_extra = rand_below(afl, afl->extras_cnt); + u32 extra_len = afl->extras[use_extra].len; + + if (unlikely(extra_len > len)) { goto retry_havoc_step; } + + u32 insert_at = rand_below(afl, len - extra_len + 1); + memcpy(buf + insert_at, afl->extras[use_extra].data, extra_len); + + break; + + } + + case MUT_EXTRA_INSERT: { + + if (unlikely(!afl->extras_cnt)) { goto retry_havoc_step; } + + u32 use_extra = rand_below(afl, afl->extras_cnt); + u32 extra_len = afl->extras[use_extra].len; + if (unlikely(len + extra_len >= max_len)) { goto retry_havoc_step; } + + u8 *ptr = afl->extras[use_extra].data; + u32 insert_at = rand_below(afl, len + 1); + + /* Tail */ + memmove(buf + insert_at + extra_len, buf + insert_at, len - insert_at); + + /* Inserted part */ + memcpy(buf + insert_at, ptr, extra_len); + len += extra_len; + + break; + + } + + case MUT_AUTO_EXTRA_OVERWRITE: { + + if (unlikely(!afl->a_extras_cnt)) { goto retry_havoc_step; } + + /* Use the dictionary. */ + + u32 use_extra = rand_below(afl, afl->a_extras_cnt); + u32 extra_len = afl->a_extras[use_extra].len; + + if (unlikely(extra_len > len)) { goto retry_havoc_step; } + + u32 insert_at = rand_below(afl, len - extra_len + 1); + memcpy(buf + insert_at, afl->a_extras[use_extra].data, extra_len); + + break; + + } + + case MUT_AUTO_EXTRA_INSERT: { + + if (unlikely(!afl->a_extras_cnt)) { goto retry_havoc_step; } + + u32 use_extra = rand_below(afl, afl->a_extras_cnt); + u32 extra_len = afl->a_extras[use_extra].len; + if (unlikely(len + extra_len >= max_len)) { goto retry_havoc_step; } + + u8 *ptr = afl->a_extras[use_extra].data; + u32 insert_at = rand_below(afl, len + 1); + + /* Tail */ + memmove(buf + insert_at + extra_len, buf + insert_at, len - insert_at); + + /* Inserted part */ + memcpy(buf + insert_at, ptr, extra_len); + len += extra_len; + + break; + + } + + case MUT_SPLICE_OVERWRITE: { + + if (unlikely(!splice_buf || !splice_len)) { goto retry_havoc_step; } + + /* overwrite mode */ + + u32 copy_from, copy_to, copy_len; + + copy_len = choose_block_len(afl, splice_len - 1); + + if (copy_len > len) copy_len = len; + + copy_from = rand_below(afl, splice_len - copy_len + 1); + copy_to = rand_below(afl, len - copy_len + 1); + memmove(buf + copy_to, splice_buf + copy_from, copy_len); + + break; + + } + + case MUT_SPLICE_INSERT: { + + if (unlikely(!splice_buf || !splice_len)) { goto retry_havoc_step; } + + if (unlikely(len + HAVOC_BLK_XL >= max_len)) { goto retry_havoc_step; } + + /* insert mode */ + + u32 clone_from, clone_to, clone_len; + + clone_len = choose_block_len(afl, splice_len); + clone_from = rand_below(afl, splice_len - clone_len + 1); + clone_to = rand_below(afl, len + 1); + + /* Head */ + + memcpy(tmp_buf, buf, clone_to); + + /* Inserted part */ + + memcpy(tmp_buf + clone_to, splice_buf + clone_from, clone_len); + + /* Tail */ + memcpy(tmp_buf + clone_to + clone_len, buf + clone_to, len - clone_to); + + len += clone_len; + memcpy(buf, tmp_buf, len); + + break; + + } + + } + + } + + } + + return len; + +} + +#endif /* !AFL_MUTATIONS_H */ + diff --git a/include/afl-persistent-replay.h b/include/afl-persistent-replay.h new file mode 100644 index 00000000..9e60ff9c --- /dev/null +++ b/include/afl-persistent-replay.h @@ -0,0 +1,131 @@ +#ifndef _HAVE_PERSISTENT_REPLAY_H +#define _HAVE_PERSISTENT_REPLAY_H + +#include <dirent.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <malloc.h> +#include <unistd.h> +#include <sys/stat.h> +#include <fcntl.h> + +#ifndef PATH_MAX + #define PATH_MAX 4096 +#endif + +static unsigned short int is_replay_record; +static unsigned int replay_record; +static unsigned int replay_record_cnt; +static char replay_record_path[PATH_MAX]; +static char *replay_record_dir; +static struct dirent **record_list; + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE +static char **record_arg = NULL; +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + +static int select_files(const struct dirent *dirbuf) { + + char fn[PATH_MAX]; + + if (dirbuf->d_name[0] == '.') { + + return 0; + + } else { + + snprintf(fn, sizeof(fn), "RECORD:%06u", replay_record); + return !!strstr(dirbuf->d_name, fn); + + } + +} + +static int compare_files(const struct dirent **da, const struct dirent **db) { + + unsigned int c1 = 0, c2 = 0; + + sscanf((*da)->d_name, "RECORD:%*u,cnt:%06u", &c1); + sscanf((*db)->d_name, "RECORD:%*u,cnt:%06u", &c2); + + return c1 - c2; + +} + +__attribute__((destructor)) static void __afl_record_replay_destroy(void) { + + for (int i = 0; i < replay_record_cnt; i++) { + + free(record_list[i]); + + } + + free(record_list); + +} + +__attribute__((constructor)) static void __afl_record_replay_init( +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + int argc, char **argv +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE +) { + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + char **argp; +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + + struct stat sb; + + /* caveat: if harness uses @@ and we don't pass it, it will regardless loop + * the number of iterations defined for AFL_LOOP (on the same file)*/ + if (!(is_replay_record = !!getenv("AFL_PERSISTENT_REPLAY"))) { + + // printf("[warning] AFL_PERSISTENT_REPLAY not set.\n"); + return; + + } + + replay_record = atoi(getenv("AFL_PERSISTENT_REPLAY")); + replay_record_dir = getenv("AFL_PERSISTENT_DIR"); + + if (!(stat(replay_record_dir, &sb) == 0 && S_ISDIR(sb.st_mode))) { + + fprintf(stderr, "[error] Can't find the requested record directory!\n"); + is_replay_record = 0; + return; + + } + + replay_record_cnt = scandir(replay_record_dir ? replay_record_dir : "./", + &record_list, select_files, compare_files); + + if (!replay_record_cnt) { + + fprintf(stderr, "[error] Can't find the requested record!\n"); + is_replay_record = 0; + + } + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + argp = argv; + while (*argp) { + + if (!strcmp(*argp, "@@")) { + + record_arg = argp; + *record_arg = replay_record_path; + break; + + } + + ++argp; + + } + +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + +} + +#endif // _HAVE_PERSISTENT_REPLAY_H + diff --git a/include/afl-prealloc.h b/include/afl-prealloc.h index d19a7b52..bcccb6b4 100644 --- a/include/afl-prealloc.h +++ b/include/afl-prealloc.h @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/afl-record-compat.h b/include/afl-record-compat.h new file mode 100644 index 00000000..3e5d60e3 --- /dev/null +++ b/include/afl-record-compat.h @@ -0,0 +1,67 @@ +#ifndef _HAVE_AFL_COMPAT_H +#define _HAVE_AFL_COMPAT_H + +#include <afl-persistent-replay.h> + +#define FUZZ_BUF_SIZE 1024000 + +// extern ssize_t read(int fildes, void *buf, size_t nbyte); + +// extern int __afl_persistent_loop(unsigned int max_cnt); +// extern unsigned char fuzz_buf[]; + +#ifndef __AFL_HAVE_MANUAL_CONTROL + #define __AFL_HAVE_MANUAL_CONTROL +#endif + +#define __AFL_FUZZ_TESTCASE_LEN (read(0, fuzz_buf, FUZZ_BUF_SIZE)) +#define __AFL_FUZZ_TESTCASE_BUF fuzz_buf +#define __AFL_FUZZ_INIT() void sync(void); +#define __AFL_INIT() sync() +#define __AFL_LOOP(x) __afl_persistent_loop(x) + +unsigned char fuzz_buf[FUZZ_BUF_SIZE]; + +int __afl_persistent_loop(unsigned int max_cnt) { + + static unsigned int cycle_cnt = 1; + static unsigned short int inited = 0; + char tcase[PATH_MAX]; + + if (is_replay_record && cycle_cnt) { + + if (!inited) { + + cycle_cnt = replay_record_cnt; + inited = 1; + + } + + snprintf(tcase, PATH_MAX, "%s/%s", + replay_record_dir ? replay_record_dir : "./", + record_list[replay_record_cnt - cycle_cnt]->d_name); + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + if (record_arg) { + + *record_arg = tcase; + + } else + +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + { + + int fd = open(tcase, O_RDONLY); + dup2(fd, 0); + close(fd); + + } + + } + + return cycle_cnt--; + +} + +#endif // _HAVE_AFL_COMPAT_H + diff --git a/include/alloc-inl.h b/include/alloc-inl.h index 1e9a192b..dad0652f 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -322,7 +322,7 @@ static inline void DFL_ck_free(void *mem) { static inline void *DFL_ck_realloc(void *orig, u32 size) { void *ret; - u32 old_size = 0; + u32 old_size = 0; if (!size) { @@ -392,7 +392,7 @@ static inline void *DFL_ck_realloc(void *orig, u32 size) { static inline u8 *DFL_ck_strdup(u8 *str) { void *ret; - u32 size; + u32 size; if (!str) return NULL; @@ -438,14 +438,14 @@ struct TRK_obj { void *ptr; char *file, *func; - u32 line; + u32 line; }; #ifdef AFL_MAIN struct TRK_obj *TRK[ALLOC_BUCKETS]; -u32 TRK_cnt[ALLOC_BUCKETS]; +u32 TRK_cnt[ALLOC_BUCKETS]; #define alloc_report() TRK_report() diff --git a/include/android-ashmem.h b/include/android-ashmem.h index 1bfd3220..065c213b 100644 --- a/include/android-ashmem.h +++ b/include/android-ashmem.h @@ -2,7 +2,9 @@ #ifndef _ANDROID_ASHMEM_H #define _ANDROID_ASHMEM_H - #define _GNU_SOURCE + #ifndef _GNU_SOURCE + #define _GNU_SOURCE + #endif #include <sys/syscall.h> #include <unistd.h> #include <fcntl.h> diff --git a/include/cmplog.h b/include/cmplog.h index 6e16e6b0..a4449a60 100644 --- a/include/cmplog.h +++ b/include/cmplog.h @@ -7,12 +7,12 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -34,22 +34,19 @@ #define CMP_MAP_W 65536 #define CMP_MAP_H 32 -#define CMP_MAP_RTN_H (CMP_MAP_H / 4) +#define CMP_MAP_RTN_H (CMP_MAP_H / 2) #define SHAPE_BYTES(x) (x + 1) -#define CMP_TYPE_INS 1 -#define CMP_TYPE_RTN 2 +#define CMP_TYPE_INS 0 +#define CMP_TYPE_RTN 1 -struct cmp_header { +struct cmp_header { // 16 bit = 2 bytes - unsigned hits : 24; - unsigned id : 24; - unsigned shape : 5; - unsigned type : 2; - unsigned attribute : 4; - unsigned overflow : 1; - unsigned reserved : 4; + unsigned hits : 6; // up to 63 entries, we have CMP_MAP_H = 32 + unsigned shape : 5; // 31+1 bytes max + unsigned type : 1; // 2: cmp, rtn + unsigned attribute : 4; // 16 for arithmetic comparison types } __attribute__((packed)); @@ -59,14 +56,17 @@ struct cmp_operands { u64 v1; u64 v0_128; u64 v1_128; + u64 unused; + u8 unused1; + u8 unused2; } __attribute__((packed)); struct cmpfn_operands { - u8 v0[31]; + u8 v0[32]; u8 v0_len; - u8 v1[31]; + u8 v1[32]; u8 v1_len; } __attribute__((packed)); diff --git a/include/common.h b/include/common.h index 8d85d201..a78dd60a 100644 --- a/include/common.h +++ b/include/common.h @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -115,6 +115,11 @@ u8 *stringify_mem_size(u8 *buf, size_t len, u64 val); u8 *stringify_time_diff(u8 *buf, size_t len, u64 cur_ms, u64 event_ms); +/* Unsafe describe time delta as simple string. + Returns a pointer to buf for convenience. */ + +u8 *u_simplestring_time_diff(u8 *buf, u64 cur_ms, u64 event_ms); + /* Unsafe Describe integer. The buf sizes are not checked. This is unsafe but fast. Will return buf for convenience. */ diff --git a/include/config.h b/include/config.h index 764c29dc..3727dab1 100644 --- a/include/config.h +++ b/include/config.h @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, - Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> + Andrea Fioraldi <andreafioraldi@gmail.com>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.07a" +#define VERSION "++4.21a" /****************************************************** * * @@ -43,9 +43,27 @@ Default: 8MB (defined in bytes) */ #define DEFAULT_SHMEM_SIZE (8 * 1024 * 1024) +/* Default time until when no more coverage finds are happening afl-fuzz + switches to exploitation mode. It automatically switches back when new + coverage is found. + Default: 300 (seconds) */ +#define STRATEGY_SWITCH_TIME 1000 + /* Default file permission umode when creating files (default: 0600) */ #define DEFAULT_PERMISSION 0600 +/* SkipDet's global configuration */ + +#define MINIMAL_BLOCK_SIZE 64 +#define SMALL_DET_TIME (60 * 1000 * 1000U) +#define MAXIMUM_INF_EXECS (16 * 1024U) +#define MAXIMUM_QUICK_EFF_EXECS (64 * 1024U) +#define THRESHOLD_DEC_TIME (20 * 60 * 1000U) + +/* Set the Prob of selecting eff_bytes 3 times more than original, + Now disabled */ +#define EFF_HAVOC_RATE 3 + /* CMPLOG/REDQUEEN TUNING * * Here you can modify tuning and solving options for CMPLOG. @@ -54,10 +72,6 @@ * */ -/* if TRANSFORM is enabled with '-l T', this additionally enables base64 - encoding/decoding */ -// #define CMPLOG_SOLVE_TRANSFORM_BASE64 - /* If a redqueen pass finds more than one solution, try to combine them? */ #define CMPLOG_COMBINE @@ -65,10 +79,10 @@ #define CMPLOG_CORPUS_PERCENT 5U /* Number of potential positions from which we decide if cmplog becomes - useless, default 8096 */ + useless, default 12288 */ #define CMPLOG_POSITIONS_MAX (12 * 1024) -/* Maximum allowed fails per CMP value. Default: 128 */ +/* Maximum allowed fails per CMP value. Default: 96 */ #define CMPLOG_FAIL_MAX 96 /* -------------------------------------*/ @@ -81,7 +95,12 @@ will be kept and written to the crash/ directory as RECORD:... files. Note that every crash will be written, not only unique ones! */ -//#define AFL_PERSISTENT_RECORD +// #define AFL_PERSISTENT_RECORD + +/* Adds support in compiler-rt to replay persistent records in @@-style + * harnesses */ + +// #define AFL_PERSISTENT_REPLAY_ARGPARSE /* console output colors: There are three ways to configure its behavior * 1. default: colored outputs fixed on: defined USE_COLOR && defined @@ -118,9 +137,9 @@ // #define _WANT_ORIGINAL_AFL_ALLOC -/* Comment out to disable fancy ANSI boxes and use poor man's 7-bit UI: */ +/* Comment out to disable fancy boxes and use poor man's 7-bit UI: */ -#ifndef ANDROID_DISABLE_FANCY // Fancy boxes are ugly from adb +#ifndef DISABLE_FANCY #define FANCY_BOXES #endif @@ -354,9 +373,10 @@ 65535, /* Overflow unsig 16-bit when incremented */ \ 65536, /* Overflow unsig 16 bit */ \ 100663045, /* Large positive number (endian-agnostic) */ \ + 2139095040, /* float infinite */ \ 2147483647 /* Overflow signed 32-bit when incremented */ -#define INTERESTING_32_LEN 8 +#define INTERESTING_32_LEN 9 /*********************************************************** * * @@ -440,7 +460,15 @@ after changing this - otherwise, SEGVs may ensue. */ #define MAP_SIZE_POW2 16 + +/* Do not change this unless you really know what you are doing. */ + #define MAP_SIZE (1U << MAP_SIZE_POW2) +#if MAP_SIZE <= 2097152 + #define MAP_INITIAL_SIZE (2 << 20) // = 2097152 +#else + #define MAP_INITIAL_SIZE MAP_SIZE +#endif /* Maximum allocator request size (keep well under INT_MAX): */ diff --git a/include/debug.h b/include/debug.h index cd621a72..5496135c 100644 --- a/include/debug.h +++ b/include/debug.h @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -116,7 +116,7 @@ * Box drawing sequences * *************************/ -#ifdef FANCY_BOXES +#ifdef FANCY_BOXES_NO_UTF #define SET_G1 "\x1b)0" /* Set G1 for box drawing */ #define RESET_G1 "\x1b)B" /* Reset G1 to ASCII */ @@ -136,22 +136,43 @@ #else - #define SET_G1 "" - #define RESET_G1 "" - #define bSTART "" - #define bSTOP "" - #define bH "-" - #define bV "|" - #define bLT "+" - #define bRT "+" - #define bLB "+" - #define bRB "+" - #define bX "+" - #define bVR "+" - #define bVL "+" - #define bHT "+" - #define bHB "+" - + #ifdef FANCY_BOXES + + #define SET_G1 "" + #define RESET_G1 "" + #define bSTART "" + #define bSTOP "" + #define bH "\u2500" /* Horizontal line */ + #define bV "\u2502" /* Vertical line */ + #define bLT "\u250c" /* Left top corner */ + #define bRT "\u2510" /* Right top corner */ + #define bLB "\u2514" /* Left bottom corner */ + #define bRB "\u2518" /* Right bottom corner */ + #define bX "\u253c" /* Cross */ + #define bVR "\u251c" /* Vertical, branch right */ + #define bVL "\u2524" /* Vertical, branch left */ + #define bHT "\u2534" /* Horizontal, branch top */ + #define bHB "\u252c" /* Horizontal, branch bottom */ + + #else + + #define SET_G1 "" + #define RESET_G1 "" + #define bSTART "" + #define bSTOP "" + #define bH "-" + #define bV "|" + #define bLT "+" + #define bRT "+" + #define bLB "+" + #define bRB "+" + #define bX "+" + #define bVR "+" + #define bVL "+" + #define bHT "+" + #define bHB "+" + + #endif #endif /* ^FANCY_BOXES */ /*********************** diff --git a/include/envs.h b/include/envs.h index edfd06e4..5b516905 100644 --- a/include/envs.h +++ b/include/envs.h @@ -16,240 +16,107 @@ static char *afl_environment_deprecated[] = { static char *afl_environment_variables[] = { - "AFL_ALIGNED_ALLOC", - "AFL_ALLOW_TMP", - "AFL_ANALYZE_HEX", - "AFL_AS", - "AFL_AUTORESUME", - "AFL_AS_FORCE_INSTRUMENT", - "AFL_BENCH_JUST_ONE", - "AFL_BENCH_UNTIL_CRASH", - "AFL_CAL_FAST", - "AFL_CC", - "AFL_CC_COMPILER", - "AFL_CMIN_ALLOW_ANY", - "AFL_CMIN_CRASHES_ONLY", - "AFL_CMPLOG_ONLY_NEW", - "AFL_CODE_END", - "AFL_CODE_START", - "AFL_COMPCOV_BINNAME", - "AFL_COMPCOV_LEVEL", - "AFL_CRASH_EXITCODE", - "AFL_CUSTOM_MUTATOR_LIBRARY", - "AFL_CUSTOM_MUTATOR_ONLY", - "AFL_CUSTOM_INFO_PROGRAM", - "AFL_CUSTOM_INFO_PROGRAM_ARGV", - "AFL_CUSTOM_INFO_PROGRAM_INPUT", - "AFL_CUSTOM_INFO_OUT", - "AFL_CXX", - "AFL_CYCLE_SCHEDULES", - "AFL_DEBUG", - "AFL_DEBUG_CHILD", - "AFL_DEBUG_GDB", - "AFL_DEBUG_UNICORN", - "AFL_DISABLE_TRIM", - "AFL_DISABLE_LLVM_INSTRUMENTATION", - "AFL_DONT_OPTIMIZE", - "AFL_DRIVER_STDERR_DUPLICATE_FILENAME", - "AFL_DUMB_FORKSRV", - "AFL_EARLY_FORKSERVER", - "AFL_ENTRYPOINT", - "AFL_EXIT_WHEN_DONE", - "AFL_EXIT_ON_TIME", - "AFL_EXIT_ON_SEED_ISSUES", - "AFL_FAST_CAL", - "AFL_FORCE_UI", - "AFL_FRIDA_DEBUG_MAPS", - "AFL_FRIDA_DRIVER_NO_HOOK", - "AFL_FRIDA_EXCLUDE_RANGES", - "AFL_FRIDA_INST_CACHE_SIZE", - "AFL_FRIDA_INST_COVERAGE_ABSOLUTE", - "AFL_FRIDA_INST_COVERAGE_FILE", - "AFL_FRIDA_INST_DEBUG_FILE", - "AFL_FRIDA_INST_INSN", - "AFL_FRIDA_INST_JIT", - "AFL_FRIDA_INST_NO_CACHE", - "AFL_FRIDA_INST_NO_DYNAMIC_LOAD", - "AFL_FRIDA_INST_NO_OPTIMIZE", - "AFL_FRIDA_INST_NO_PREFETCH", - "AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH", + "AFL_ALIGNED_ALLOC", "AFL_ALLOW_TMP", "AFL_ANALYZE_HEX", "AFL_AS", + "AFL_AUTORESUME", "AFL_AS_FORCE_INSTRUMENT", "AFL_BENCH_JUST_ONE", + "AFL_BENCH_UNTIL_CRASH", "AFL_CAL_FAST", "AFL_CC", "AFL_CC_COMPILER", + "AFL_CMIN_ALLOW_ANY", "AFL_CMIN_CRASHES_ONLY", "AFL_CMPLOG_ONLY_NEW", + "AFL_CODE_END", "AFL_CODE_START", "AFL_COMPCOV_BINNAME", + "AFL_DUMP_CYCLOMATIC_COMPLEXITY", "AFL_CMPLOG_MAX_LEN", "AFL_COMPCOV_LEVEL", + "AFL_CRASH_EXITCODE", "AFL_CRASHING_SEEDS_AS_NEW_CRASH", + "AFL_CUSTOM_MUTATOR_LIBRARY", "AFL_CUSTOM_MUTATOR_ONLY", + "AFL_CUSTOM_INFO_PROGRAM", "AFL_CUSTOM_INFO_PROGRAM_ARGV", + "AFL_CUSTOM_INFO_PROGRAM_INPUT", "AFL_CUSTOM_INFO_OUT", "AFL_CXX", + "AFL_CYCLE_SCHEDULES", "AFL_DEBUG", "AFL_DEBUG_CHILD", "AFL_DEBUG_GDB", + "AFL_DEBUG_UNICORN", "AFL_DISABLE_REDUNDANT", "AFL_NO_REDUNDANT", + "AFL_DISABLE_TRIM", "AFL_NO_TRIM", "AFL_DISABLE_LLVM_INSTRUMENTATION", + "AFL_DONT_OPTIMIZE", "AFL_DRIVER_STDERR_DUPLICATE_FILENAME", + "AFL_DUMB_FORKSRV", "AFL_EARLY_FORKSERVER", "AFL_ENTRYPOINT", + "AFL_EXIT_WHEN_DONE", "AFL_EXIT_ON_TIME", "AFL_EXIT_ON_SEED_ISSUES", + "AFL_FAST_CAL", "AFL_FINAL_SYNC", "AFL_FORCE_UI", "AFL_FRIDA_DEBUG_MAPS", + "AFL_FRIDA_DRIVER_NO_HOOK", "AFL_FRIDA_EXCLUDE_RANGES", + "AFL_FRIDA_INST_CACHE_SIZE", "AFL_FRIDA_INST_COVERAGE_ABSOLUTE", + "AFL_FRIDA_INST_COVERAGE_FILE", "AFL_FRIDA_INST_DEBUG_FILE", + "AFL_FRIDA_INST_INSN", "AFL_FRIDA_INST_JIT", "AFL_FRIDA_INST_NO_CACHE", + "AFL_FRIDA_INST_NO_DYNAMIC_LOAD", "AFL_FRIDA_INST_NO_OPTIMIZE", + "AFL_FRIDA_INST_NO_PREFETCH", "AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH", "AFL_FRIDA_INST_NO_SUPPRESS" "AFL_FRIDA_INST_RANGES", - "AFL_FRIDA_INST_REGS_FILE", - "AFL_FRIDA_INST_SEED", - "AFL_FRIDA_INST_TRACE", - "AFL_FRIDA_INST_TRACE_UNIQUE", - "AFL_FRIDA_INST_UNSTABLE_COVERAGE_FILE", - "AFL_FRIDA_JS_SCRIPT", - "AFL_FRIDA_OUTPUT_STDOUT", - "AFL_FRIDA_OUTPUT_STDERR", - "AFL_FRIDA_PERSISTENT_ADDR", - "AFL_FRIDA_PERSISTENT_CNT", - "AFL_FRIDA_PERSISTENT_DEBUG", - "AFL_FRIDA_PERSISTENT_HOOK", - "AFL_FRIDA_PERSISTENT_RET", - "AFL_FRIDA_STALKER_ADJACENT_BLOCKS", - "AFL_FRIDA_STALKER_IC_ENTRIES", - "AFL_FRIDA_STALKER_NO_BACKPATCH", - "AFL_FRIDA_STATS_FILE", - "AFL_FRIDA_STATS_INTERVAL", - "AFL_FRIDA_TRACEABLE", + "AFL_FRIDA_INST_REGS_FILE", "AFL_FRIDA_INST_SEED", "AFL_FRIDA_INST_TRACE", + "AFL_FRIDA_INST_TRACE_UNIQUE", "AFL_FRIDA_INST_UNSTABLE_COVERAGE_FILE", + "AFL_FRIDA_JS_SCRIPT", "AFL_FRIDA_OUTPUT_STDOUT", "AFL_FRIDA_OUTPUT_STDERR", + "AFL_FRIDA_PERSISTENT_ADDR", "AFL_FRIDA_PERSISTENT_CNT", + "AFL_FRIDA_PERSISTENT_DEBUG", "AFL_FRIDA_PERSISTENT_HOOK", + "AFL_FRIDA_PERSISTENT_RET", "AFL_FRIDA_STALKER_ADJACENT_BLOCKS", + "AFL_FRIDA_STALKER_IC_ENTRIES", "AFL_FRIDA_STALKER_NO_BACKPATCH", + "AFL_FRIDA_STATS_FILE", "AFL_FRIDA_STATS_INTERVAL", "AFL_FRIDA_TRACEABLE", "AFL_FRIDA_VERBOSE", "AFL_FUZZER_ARGS", // oss-fuzz - "AFL_FUZZER_STATS_UPDATE_INTERVAL", - "AFL_GDB", - "AFL_GCC_ALLOWLIST", - "AFL_GCC_DENYLIST", - "AFL_GCC_BLOCKLIST", - "AFL_GCC_INSTRUMENT_FILE", - "AFL_GCC_OUT_OF_LINE", - "AFL_GCC_SKIP_NEVERZERO", - "AFL_GCJ", - "AFL_HANG_TMOUT", - "AFL_FORKSRV_INIT_TMOUT", - "AFL_HARDEN", - "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", - "AFL_IGNORE_PROBLEMS", - "AFL_IGNORE_PROBLEMS_COVERAGE", - "AFL_IGNORE_TIMEOUTS", - "AFL_IGNORE_UNKNOWN_ENVS", - "AFL_IMPORT_FIRST", - "AFL_INPUT_LEN_MIN", - "AFL_INPUT_LEN_MAX", - "AFL_INST_LIBS", - "AFL_INST_RATIO", - "AFL_KEEP_TIMEOUTS", - "AFL_KILL_SIGNAL", - "AFL_FORK_SERVER_KILL_SIGNAL", - "AFL_KEEP_TRACES", - "AFL_KEEP_ASSEMBLY", - "AFL_LD_HARD_FAIL", - "AFL_LD_LIMIT_MB", - "AFL_LD_NO_CALLOC_OVER", - "AFL_LD_PASSTHROUGH", - "AFL_REAL_LD", - "AFL_LD_PRELOAD", - "AFL_LD_VERBOSE", - "AFL_LLVM_ALLOWLIST", - "AFL_LLVM_DENYLIST", - "AFL_LLVM_BLOCKLIST", - "AFL_CMPLOG", - "AFL_LLVM_CMPLOG", - "AFL_GCC_CMPLOG", - "AFL_LLVM_INSTRIM", - "AFL_LLVM_CALLER", - "AFL_LLVM_CTX", - "AFL_LLVM_CTX_K", - "AFL_LLVM_DICT2FILE", - "AFL_LLVM_DICT2FILE_NO_MAIN", - "AFL_LLVM_DOCUMENT_IDS", - "AFL_LLVM_INSTRIM_LOOPHEAD", - "AFL_LLVM_INSTRUMENT", - "AFL_LLVM_LTO_AUTODICTIONARY", - "AFL_LLVM_AUTODICTIONARY", + "AFL_FUZZER_STATS_UPDATE_INTERVAL", "AFL_GDB", "AFL_GCC_ALLOWLIST", + "AFL_GCC_DENYLIST", "AFL_GCC_BLOCKLIST", "AFL_GCC_INSTRUMENT_FILE", + "AFL_GCC_OUT_OF_LINE", "AFL_GCC_SKIP_NEVERZERO", "AFL_GCJ", + "AFL_HANG_TMOUT", "AFL_FORKSRV_INIT_TMOUT", "AFL_HARDEN", + "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", "AFL_IGNORE_PROBLEMS", + "AFL_IGNORE_PROBLEMS_COVERAGE", "AFL_IGNORE_SEED_PROBLEMS", + "AFL_IGNORE_TIMEOUTS", "AFL_IGNORE_UNKNOWN_ENVS", "AFL_IMPORT_FIRST", + "AFL_INPUT_LEN_MIN", "AFL_INPUT_LEN_MAX", "AFL_INST_LIBS", "AFL_INST_RATIO", + "AFL_KEEP_TIMEOUTS", "AFL_KILL_SIGNAL", "AFL_FORK_SERVER_KILL_SIGNAL", + "AFL_KEEP_TRACES", "AFL_KEEP_ASSEMBLY", "AFL_LD_HARD_FAIL", + "AFL_LD_LIMIT_MB", "AFL_LD_NO_CALLOC_OVER", "AFL_LD_PASSTHROUGH", + "AFL_REAL_LD", "AFL_LD_PRELOAD", "AFL_LD_VERBOSE", "AFL_LLVM_ALLOWLIST", + "AFL_LLVM_DENYLIST", "AFL_LLVM_BLOCKLIST", "AFL_CMPLOG", "AFL_LLVM_CMPLOG", + "AFL_GCC_CMPLOG", "AFL_LLVM_INSTRIM", "AFL_LLVM_CALLER", "AFL_LLVM_CTX", + "AFL_LLVM_LTO_CALLER", "AFL_LLVM_LTO_CTX", "AFL_LLVM_LTO_CALLER_DEPTH", + "AFL_LLVM_LTO_CTX_DEPTH", "AFL_LLVM_CALLER_DEPTH", "AFL_LLVM_CTX_DEPTH", + "AFL_LLVM_CTX_K", "AFL_LLVM_DICT2FILE", "AFL_LLVM_DICT2FILE_NO_MAIN", + "AFL_LLVM_DOCUMENT_IDS", "AFL_LLVM_INSTRIM_LOOPHEAD", "AFL_LLVM_INSTRUMENT", + "AFL_LLVM_LTO_AUTODICTIONARY", "AFL_LLVM_AUTODICTIONARY", "AFL_LLVM_SKIPSINGLEBLOCK", - "AFL_LLVM_INSTRIM_SKIPSINGLEBLOCK", - "AFL_LLVM_LAF_SPLIT_COMPARES", - "AFL_LLVM_LAF_SPLIT_COMPARES_BITW", - "AFL_LLVM_LAF_SPLIT_FLOATS", - "AFL_LLVM_LAF_SPLIT_SWITCHES", - "AFL_LLVM_LAF_ALL", - "AFL_LLVM_LAF_TRANSFORM_COMPARES", - "AFL_LLVM_MAP_ADDR", - "AFL_LLVM_MAP_DYNAMIC", - "AFL_LLVM_NGRAM_SIZE", - "AFL_NGRAM_SIZE", - "AFL_LLVM_NOT_ZERO", - "AFL_LLVM_INSTRUMENT_FILE", - "AFL_LLVM_THREADSAFE_INST", - "AFL_LLVM_SKIP_NEVERZERO", - "AFL_NO_AFFINITY", - "AFL_TRY_AFFINITY", - "AFL_LLVM_LTO_DONTWRITEID", + // Marker: ADD_TO_INJECTIONS + "AFL_LLVM_INJECTIONS_ALL", "AFL_LLVM_INJECTIONS_SQL", + "AFL_LLVM_INJECTIONS_LDAP", "AFL_LLVM_INJECTIONS_XSS", + "AFL_LLVM_INSTRIM_SKIPSINGLEBLOCK", "AFL_LLVM_LAF_SPLIT_COMPARES", + "AFL_LLVM_LAF_SPLIT_COMPARES_BITW", "AFL_LLVM_LAF_SPLIT_FLOATS", + "AFL_LLVM_LAF_SPLIT_SWITCHES", "AFL_LLVM_LAF_ALL", + "AFL_LLVM_LAF_TRANSFORM_COMPARES", "AFL_LLVM_MAP_ADDR", + "AFL_LLVM_MAP_DYNAMIC", "AFL_LLVM_NGRAM_SIZE", "AFL_NGRAM_SIZE", + "AFL_LLVM_NO_RPATH", "AFL_LLVM_NOT_ZERO", "AFL_LLVM_INSTRUMENT_FILE", + "AFL_LLVM_THREADSAFE_INST", "AFL_LLVM_SKIP_NEVERZERO", "AFL_NO_AFFINITY", + "AFL_TRY_AFFINITY", "AFL_LLVM_LTO_DONTWRITEID", "AFL_LLVM_LTO_SKIPINIT" "AFL_LLVM_LTO_STARTID", - "AFL_NO_ARITH", - "AFL_NO_AUTODICT", - "AFL_NO_BUILTIN", + "AFL_FUZZER_LOOPCOUNT", "AFL_NO_ARITH", "AFL_NO_AUTODICT", "AFL_NO_BUILTIN", #if defined USE_COLOR && !defined ALWAYS_COLORED - "AFL_NO_COLOR", - "AFL_NO_COLOUR", + "AFL_NO_COLOR", "AFL_NO_COLOUR", #endif "AFL_NO_CPU_RED", - "AFL_NO_CRASH_README", - "AFL_NO_FORKSRV", - "AFL_NO_UI", - "AFL_NO_PYTHON", - "AFL_NO_STARTUP_CALIBRATION", - "AFL_NO_WARN_INSTABILITY", - "AFL_UNTRACER_FILE", - "AFL_LLVM_USE_TRACE_PC", - "AFL_MAP_SIZE", - "AFL_MAPSIZE", + "AFL_NO_CFG_FUZZING", // afl.rs rust crate option + "AFL_NO_CRASH_README", "AFL_NO_FORKSRV", "AFL_NO_UI", "AFL_NO_PYTHON", + "AFL_NO_STARTUP_CALIBRATION", "AFL_NO_WARN_INSTABILITY", + "AFL_UNTRACER_FILE", "AFL_LLVM_USE_TRACE_PC", "AFL_MAP_SIZE", "AFL_MAPSIZE", "AFL_MAX_DET_EXTRAS", "AFL_NO_X86", // not really an env but we dont want to warn on it - "AFL_NOOPT", - "AFL_PASSTHROUGH", - "AFL_PATH", - "AFL_PERFORMANCE_FILE", - "AFL_PERSISTENT_RECORD", - "AFL_POST_PROCESS_KEEP_ORIGINAL", - "AFL_PRELOAD", - "AFL_TARGET_ENV", - "AFL_PYTHON_MODULE", - "AFL_QEMU_CUSTOM_BIN", - "AFL_QEMU_COMPCOV", - "AFL_QEMU_COMPCOV_DEBUG", - "AFL_QEMU_DEBUG_MAPS", - "AFL_QEMU_DISABLE_CACHE", - "AFL_QEMU_DRIVER_NO_HOOK", - "AFL_QEMU_FORCE_DFL", - "AFL_QEMU_PERSISTENT_ADDR", - "AFL_QEMU_PERSISTENT_CNT", - "AFL_QEMU_PERSISTENT_GPR", - "AFL_QEMU_PERSISTENT_HOOK", - "AFL_QEMU_PERSISTENT_MEM", - "AFL_QEMU_PERSISTENT_RET", - "AFL_QEMU_PERSISTENT_RETADDR_OFFSET", - "AFL_QEMU_PERSISTENT_EXITS", - "AFL_QEMU_INST_RANGES", - "AFL_QEMU_EXCLUDE_RANGES", - "AFL_QEMU_SNAPSHOT", - "AFL_QEMU_TRACK_UNSTABLE", - "AFL_QUIET", - "AFL_RANDOM_ALLOC_CANARY", - "AFL_REAL_PATH", - "AFL_SHUFFLE_QUEUE", - "AFL_SKIP_BIN_CHECK", - "AFL_SKIP_CPUFREQ", - "AFL_SKIP_CRASHES", - "AFL_SKIP_OSSFUZZ", - "AFL_STATSD", - "AFL_STATSD_HOST", - "AFL_STATSD_PORT", - "AFL_STATSD_TAGS_FLAVOR", - "AFL_SYNC_TIME", - "AFL_TESTCACHE_SIZE", - "AFL_TESTCACHE_ENTRIES", - "AFL_TMIN_EXACT", - "AFL_TMPDIR", - "AFL_TOKEN_FILE", - "AFL_TRACE_PC", - "AFL_USE_ASAN", - "AFL_USE_MSAN", - "AFL_USE_TRACE_PC", - "AFL_USE_UBSAN", - "AFL_USE_TSAN", - "AFL_USE_CFISAN", - "AFL_USE_LSAN", - "AFL_WINE_PATH", - "AFL_NO_SNAPSHOT", - "AFL_EXPAND_HAVOC_NOW", - "AFL_USE_FASAN", - "AFL_USE_QASAN", - "AFL_PRINT_FILENAMES", - "AFL_PIZZA_MODE", - NULL + "AFL_NOOPT", "AFL_NYX_AUX_SIZE", "AFL_NYX_DISABLE_SNAPSHOT_MODE", + "AFL_NYX_HANDLE_INVALID_WRITE", "AFL_NYX_LOG", "AFL_NYX_REUSE_SNAPSHOT", + "AFL_PASSTHROUGH", "AFL_PATH", "AFL_PERFORMANCE_FILE", + "AFL_PERSISTENT_RECORD", "AFL_POST_PROCESS_KEEP_ORIGINAL", "AFL_PRELOAD", + "AFL_TARGET_ENV", "AFL_PYTHON_MODULE", "AFL_QEMU_CUSTOM_BIN", + "AFL_QEMU_COMPCOV", "AFL_QEMU_COMPCOV_DEBUG", "AFL_QEMU_DEBUG_MAPS", + "AFL_QEMU_DISABLE_CACHE", "AFL_QEMU_DRIVER_NO_HOOK", "AFL_QEMU_FORCE_DFL", + "AFL_QEMU_PERSISTENT_ADDR", "AFL_QEMU_PERSISTENT_CNT", + "AFL_QEMU_PERSISTENT_GPR", "AFL_QEMU_PERSISTENT_HOOK", + "AFL_QEMU_PERSISTENT_MEM", "AFL_QEMU_PERSISTENT_RET", + "AFL_QEMU_PERSISTENT_RETADDR_OFFSET", "AFL_QEMU_PERSISTENT_EXITS", + "AFL_QEMU_INST_RANGES", "AFL_QEMU_EXCLUDE_RANGES", "AFL_QEMU_SNAPSHOT", + "AFL_QEMU_TRACK_UNSTABLE", "AFL_QUIET", "AFL_RANDOM_ALLOC_CANARY", + "AFL_REAL_PATH", "AFL_SHA1_FILENAMES", "AFL_SHUFFLE_QUEUE", + "AFL_SKIP_BIN_CHECK", "AFL_SKIP_CPUFREQ", "AFL_SKIP_CRASHES", + "AFL_SKIP_OSSFUZZ", "AFL_STATSD", "AFL_STATSD_HOST", "AFL_STATSD_PORT", + "AFL_STATSD_TAGS_FLAVOR", "AFL_SYNC_TIME", "AFL_TESTCACHE_SIZE", + "AFL_TESTCACHE_ENTRIES", "AFL_TMIN_EXACT", "AFL_TMPDIR", "AFL_TOKEN_FILE", + "AFL_TRACE_PC", "AFL_USE_ASAN", "AFL_USE_MSAN", "AFL_USE_TRACE_PC", + "AFL_USE_UBSAN", "AFL_USE_TSAN", "AFL_USE_CFISAN", "AFL_USE_LSAN", + "AFL_WINE_PATH", "AFL_NO_SNAPSHOT", "AFL_EXPAND_HAVOC_NOW", "AFL_USE_FASAN", + "AFL_USE_QASAN", "AFL_PRINT_FILENAMES", "AFL_PIZZA_MODE", NULL }; diff --git a/include/forkserver.h b/include/forkserver.h index f5069ce2..593e34a2 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -7,12 +7,12 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co>> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -86,6 +86,8 @@ typedef struct { uint32_t size); bool (*nyx_remove_work_dir)(const char *workdir); + bool (*nyx_config_set_aux_buffer_size)(void *config, + uint32_t aux_buffer_size); } nyx_plugin_handler_t; @@ -124,7 +126,8 @@ typedef struct afl_forkserver { u8 *out_file, /* File to fuzz, if any */ *target_path; /* Path of the target */ - FILE *plot_file; /* Gnuplot output file */ + FILE *plot_file, /* Gnuplot output file */ + *det_plot_file; /* Note: last_run_timed_out is u32 to send it to the child as 4 byte array */ u32 last_run_timed_out; /* Traced process timed out? */ @@ -185,6 +188,8 @@ typedef struct afl_forkserver { u8 persistent_mode; + u32 max_length; + #ifdef __linux__ nyx_plugin_handler_t *nyx_handlers; char *out_dir_path; /* path to the output directory */ @@ -195,8 +200,10 @@ typedef struct afl_forkserver { u32 nyx_id; /* nyx runner id (0 -> master) */ u32 nyx_bind_cpu_id; /* nyx runner cpu id */ char *nyx_aux_string; + u32 nyx_aux_string_len; bool nyx_use_tmp_workdir; char *nyx_tmp_workdir_path; + s32 nyx_log_fd; #endif } afl_forkserver_t; diff --git a/include/hash.h b/include/hash.h index 0243c5b7..5d56a108 100644 --- a/include/hash.h +++ b/include/hash.h @@ -15,7 +15,7 @@ Other code written by Michal Zalewski Copyright 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/list.h b/include/list.h index 283bf035..bec9abbc 100644 --- a/include/list.h +++ b/include/list.h @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/sharedmem.h b/include/sharedmem.h index d32bd845..036fa560 100644 --- a/include/sharedmem.h +++ b/include/sharedmem.h @@ -7,12 +7,12 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/snapshot-inl.h b/include/snapshot-inl.h index 3864e473..e577b013 100644 --- a/include/snapshot-inl.h +++ b/include/snapshot-inl.h @@ -7,12 +7,12 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/t1ha.h b/include/t1ha.h new file mode 100644 index 00000000..1af29395 --- /dev/null +++ b/include/t1ha.h @@ -0,0 +1,738 @@ +/* + * Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com, + * Fast Positive Hash. + * + * Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>, + * The 1Hippeus project (t1h). + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgement in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * t1ha = { Fast Positive Hash, aka "ŠŠ¾Š·ŠøŃŠøŠ²Š½ŃŠ¹ Š„ŃŃ" } + * by [Positive Technologies](https://www.ptsecurity.ru) + * + * Briefly, it is a 64-bit Hash Function: + * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64, + * but portable and without penalties it can run on any 64-bit CPU. + * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash + * and all others portable hash-functions (which do not use specific + * hardware tricks). + * 3. Not suitable for cryptography. + * + * The Future will (be) Positive. ŠŃŃ Š±ŃŠ“ŠµŃ Ń Š¾ŃŠ¾ŃŠ¾. + * + * ACKNOWLEDGEMENT: + * The t1ha was originally developed by Leonid Yuriev (ŠŠµŠ¾Š½ŠøŠ“ Š®ŃŃŠµŠ²) + * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta! + */ + +#pragma once + +/***************************************************************************** + * + * PLEASE PAY ATTENTION TO THE FOLLOWING NOTES + * about macros definitions which controls t1ha behaviour and/or performance. + * + * + * 1) T1HA_SYS_UNALIGNED_ACCESS = Defines the system/platform/CPU/architecture + * abilities for unaligned data access. + * + * By default, when the T1HA_SYS_UNALIGNED_ACCESS not defined, + * it will defined on the basis hardcoded knowledge about of capabilities + * of most common CPU architectures. But you could override this + * default behavior when build t1ha library itself: + * + * // To disable unaligned access at all. + * #define T1HA_SYS_UNALIGNED_ACCESS 0 + * + * // To enable unaligned access, but indicate that it significantly slow. + * #define T1HA_SYS_UNALIGNED_ACCESS 1 + * + * // To enable unaligned access, and indicate that it effecient. + * #define T1HA_SYS_UNALIGNED_ACCESS 2 + * + * + * 2) T1HA_USE_FAST_ONESHOT_READ = Controls the data reads at the end of buffer. + * + * When defined to non-zero, t1ha will use 'one shot' method for reading + * up to 8 bytes at the end of data. In this case just the one 64-bit read + * will be performed even when the available less than 8 bytes. + * + * This is little bit faster that switching by length of data tail. + * Unfortunately this will triggering a false-positive alarms from Valgrind, + * AddressSanitizer and other similar tool. + * + * By default, t1ha defines it to 1, but you could override this + * default behavior when build t1ha library itself: + * + * // For little bit faster and small code. + * #define T1HA_USE_FAST_ONESHOT_READ 1 + * + * // For calmness if doubt. + * #define T1HA_USE_FAST_ONESHOT_READ 0 + * + * + * 3) T1HA0_RUNTIME_SELECT = Controls choice fastest function in runtime. + * + * t1ha library offers the t1ha0() function as the fastest for current CPU. + * But actual CPU's features/capabilities and may be significantly different, + * especially on x86 platform. Therefore, internally, t1ha0() may require + * dynamic dispatching for choice best implementation. + * + * By default, t1ha enables such runtime choice and (may be) corresponding + * indirect calls if it reasonable, but you could override this default + * behavior when build t1ha library itself: + * + * // To enable runtime choice of fastest implementation. + * #define T1HA0_RUNTIME_SELECT 1 + * + * // To disable runtime choice of fastest implementation. + * #define T1HA0_RUNTIME_SELECT 0 + * + * When T1HA0_RUNTIME_SELECT is nonzero the t1ha0_resolve() function could + * be used to get actual t1ha0() implementation address at runtime. This is + * useful for two cases: + * - calling by local pointer-to-function usually is little + * bit faster (less overhead) than via a PLT thru the DSO boundary. + * - GNU Indirect functions (see below) don't supported by environment + * and calling by t1ha0_funcptr is not available and/or expensive. + * + * 4) T1HA_USE_INDIRECT_FUNCTIONS = Controls usage of GNU Indirect functions. + * + * In continue of T1HA0_RUNTIME_SELECT the T1HA_USE_INDIRECT_FUNCTIONS + * controls usage of ELF indirect functions feature. In general, when + * available, this reduces overhead of indirect function's calls though + * a DSO-bundary (https://sourceware.org/glibc/wiki/GNU_IFUNC). + * + * By default, t1ha engage GNU Indirect functions when it available + * and useful, but you could override this default behavior when build + * t1ha library itself: + * + * // To enable use of GNU ELF Indirect functions. + * #define T1HA_USE_INDIRECT_FUNCTIONS 1 + * + * // To disable use of GNU ELF Indirect functions. This may be useful + * // if the actual toolchain or the system's loader don't support ones. + * #define T1HA_USE_INDIRECT_FUNCTIONS 0 + * + * 5) T1HA0_AESNI_AVAILABLE = Controls AES-NI detection and dispatching on x86. + * + * In continue of T1HA0_RUNTIME_SELECT the T1HA0_AESNI_AVAILABLE controls + * detection and usage of AES-NI CPU's feature. On the other hand, this + * requires compiling parts of t1ha library with certain properly options, + * and could be difficult or inconvenient in some cases. + * + * By default, t1ha engade AES-NI for t1ha0() on the x86 platform, but + * you could override this default behavior when build t1ha library itself: + * + * // To disable detection and usage of AES-NI instructions for t1ha0(). + * // This may be useful when you unable to build t1ha library properly + * // or known that AES-NI will be unavailable at the deploy. + * #define T1HA0_AESNI_AVAILABLE 0 + * + * // To force detection and usage of AES-NI instructions for t1ha0(), + * // but I don't known reasons to anybody would need this. + * #define T1HA0_AESNI_AVAILABLE 1 + * + * 6) T1HA0_DISABLED, T1HA1_DISABLED, T1HA2_DISABLED = Controls availability of + * t1ha functions. + * + * In some cases could be useful to import/use only few of t1ha functions + * or just the one. So, this definitions allows disable corresponding parts + * of t1ha library. + * + * // To disable t1ha0(), t1ha0_32le(), t1ha0_32be() and all AES-NI. + * #define T1HA0_DISABLED + * + * // To disable t1ha1_le() and t1ha1_be(). + * #define T1HA1_DISABLED + * + * // To disable t1ha2_atonce(), t1ha2_atonce128() and so on. + * #define T1HA2_DISABLED + * + *****************************************************************************/ + +#define T1HA_VERSION_MAJOR 2 +#define T1HA_VERSION_MINOR 1 +#define T1HA_VERSION_RELEASE 1 + +#ifndef __has_attribute + #define __has_attribute(x) (0) +#endif + +#ifndef __has_include + #define __has_include(x) (0) +#endif + +#ifndef __GNUC_PREREQ + #if defined(__GNUC__) && defined(__GNUC_MINOR__) + #define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) + #else + #define __GNUC_PREREQ(maj, min) 0 + #endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ + #ifdef __clang__ + #define __CLANG_PREREQ(maj, min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) + #else + #define __CLANG_PREREQ(maj, min) (0) + #endif +#endif /* __CLANG_PREREQ */ + +#ifndef __LCC_PREREQ + #ifdef __LCC__ + #define __LCC_PREREQ(maj, min) \ + ((__LCC__ << 16) + __LCC_MINOR__ >= ((maj) << 16) + (min)) + #else + #define __LCC_PREREQ(maj, min) (0) + #endif +#endif /* __LCC_PREREQ */ + +/*****************************************************************************/ + +#ifdef _MSC_VER + /* Avoid '16' bytes padding added after data member 't1ha_context::total' + * and other warnings from std-headers if warning-level > 3. */ + #pragma warning(push, 3) +#endif + +#if defined(__cplusplus) && __cplusplus >= 201103L + #include <climits> + #include <cstddef> + #include <cstdint> +#else + #include <limits.h> + #include <stddef.h> + #include <stdint.h> +#endif + +/*****************************************************************************/ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) + #ifndef __ia32__ + /* LY: define neutral __ia32__ for x86 and x86-64 archs */ + #define __ia32__ 1 + #endif /* __ia32__ */ + #if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) + /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ + #define __amd64__ 1 + #endif /* __amd64__ */ +#endif /* all x86 */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ + +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \ + defined(HAVE_ENDIAN_H) || __has_include(<endian.h>) +#include <endian.h> +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>) +#include <machine/endian.h> +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>) +#include <sys/isa_defs.h> +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include(<sys/types.h>) && __has_include(<sys/endian.h>)) +#include <sys/endian.h> +#include <sys/types.h> +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NETBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>) +#include <sys/param.h> +#endif /* OS */ + +/* *INDENT-ON* */ +/* clang-format on */ + + #if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) + #define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN + #define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN + #define __BYTE_ORDER__ __BYTE_ORDER + #elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) + #define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN + #define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN + #define __BYTE_ORDER__ _BYTE_ORDER + #else + #define __ORDER_LITTLE_ENDIAN__ 1234 + #define __ORDER_BIG_ENDIAN__ 4321 + + #if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || \ + defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || \ + defined(__MIPSEL) || defined(_M_ARM) || defined(_M_ARM64) || \ + defined(__e2k__) || defined(__elbrus_4c__) || \ + defined(__elbrus_8c__) || defined(__bfin__) || defined(__BFIN__) || \ + defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || \ + defined(__ia32__) || defined(__CYGWIN__) || defined(_WIN64) || \ + defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) + #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + + #elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || \ + defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || \ + defined(__MIPSEB) || defined(__m68k__) || defined(M68000) || \ + defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || \ + defined(__THW_370__) || defined(__s390__) || defined(__s390x__) || \ + defined(__SYSC_ZARCH__) + #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + + #else + #error __BYTE_ORDER__ should be defined. + #endif /* Arch */ + + #endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/*****************************************************************************/ + +#ifndef __dll_export + #if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) + #if defined(__GNUC__) || __has_attribute(dllexport) + #define __dll_export __attribute__((dllexport)) + #else + #define __dll_export __declspec(dllexport) + #endif + #elif defined(__GNUC__) || __has_attribute(__visibility__) + #define __dll_export __attribute__((__visibility__("default"))) + #else + #define __dll_export + #endif +#endif /* __dll_export */ + +#ifndef __dll_import + #if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) + #if defined(__GNUC__) || __has_attribute(dllimport) + #define __dll_import __attribute__((dllimport)) + #else + #define __dll_import __declspec(dllimport) + #endif + #elif defined(__GNUC__) || __has_attribute(__visibility__) + #define __dll_import __attribute__((__visibility__("default"))) + #else + #define __dll_import + #endif +#endif /* __dll_import */ + +#ifndef __force_inline + #ifdef _MSC_VER + #define __force_inline __forceinline + #elif __GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__) + #define __force_inline __inline __attribute__((__always_inline__)) + #else + #define __force_inline __inline + #endif +#endif /* __force_inline */ + +#ifndef T1HA_API + #if defined(t1ha_EXPORTS) + #define T1HA_API __dll_export + #elif defined(t1ha_IMPORTS) + #define T1HA_API __dll_import + #else + #define T1HA_API + #endif +#endif /* T1HA_API */ + +#if defined(_MSC_VER) && defined(__ia32__) + #define T1HA_ALIGN_PREFIX __declspec(align(32)) /* required only for SIMD */ +#else + #define T1HA_ALIGN_PREFIX +#endif /* _MSC_VER */ + +#if defined(__GNUC__) && defined(__ia32__) + #define T1HA_ALIGN_SUFFIX \ + __attribute__((__aligned__(32))) /* required only for SIMD */ +#else + #define T1HA_ALIGN_SUFFIX +#endif /* GCC x86 */ + +#ifndef T1HA_USE_INDIRECT_FUNCTIONS + /* GNU ELF indirect functions usage control. For more info please see + * https://en.wikipedia.org/wiki/Executable_and_Linkable_Format + * and https://sourceware.org/glibc/wiki/GNU_IFUNC */ + #if defined(__ELF__) && defined(__amd64__) && \ + (__has_attribute(__ifunc__) || \ + (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && \ + !defined(__SANITIZE_ADDRESS__) && !defined(__SSP_ALL__))) + /* Enable gnu_indirect_function by default if : + * - ELF AND x86_64 + * - attribute(__ifunc__) is available OR + * GCC >= 4 WITHOUT -fsanitize=address NOR -fstack-protector-all */ + #define T1HA_USE_INDIRECT_FUNCTIONS 1 + #else + #define T1HA_USE_INDIRECT_FUNCTIONS 0 + #endif +#endif /* T1HA_USE_INDIRECT_FUNCTIONS */ + +#if __GNUC_PREREQ(4, 0) + #pragma GCC visibility push(hidden) +#endif /* __GNUC_PREREQ(4,0) */ + +#ifdef __cplusplus +extern "C" { + +#endif + +typedef union T1HA_ALIGN_PREFIX t1ha_state256 { + + uint8_t bytes[32]; + uint32_t u32[8]; + uint64_t u64[4]; + struct { + + uint64_t a, b, c, d; + + } n; + +} t1ha_state256_t T1HA_ALIGN_SUFFIX; + +typedef struct t1ha_context { + + t1ha_state256_t state; + t1ha_state256_t buffer; + size_t partial; + uint64_t total; + +} t1ha_context_t; + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +/****************************************************************************** + * + * Self-testing API. + * + * Unfortunately, some compilers (exactly only Microsoft Visual C/C++) has + * a bugs which leads t1ha-functions to produce wrong results. This API allows + * check the correctness of the actual code in runtime. + * + * All check-functions returns 0 on success, or -1 in case the corresponding + * hash-function failed verification. PLEASE, always perform such checking at + * initialization of your code, if you using MSVC or other troubleful compilers. + */ + +T1HA_API int t1ha_selfcheck__all_enabled(void); + +#ifndef T1HA2_DISABLED +T1HA_API int t1ha_selfcheck__t1ha2_atonce(void); +T1HA_API int t1ha_selfcheck__t1ha2_atonce128(void); +T1HA_API int t1ha_selfcheck__t1ha2_stream(void); +T1HA_API int t1ha_selfcheck__t1ha2(void); +#endif /* T1HA2_DISABLED */ + +#ifndef T1HA1_DISABLED +T1HA_API int t1ha_selfcheck__t1ha1_le(void); +T1HA_API int t1ha_selfcheck__t1ha1_be(void); +T1HA_API int t1ha_selfcheck__t1ha1(void); +#endif /* T1HA1_DISABLED */ + +#ifndef T1HA0_DISABLED +T1HA_API int t1ha_selfcheck__t1ha0_32le(void); +T1HA_API int t1ha_selfcheck__t1ha0_32be(void); +T1HA_API int t1ha_selfcheck__t1ha0(void); + + /* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */ + #ifndef T1HA0_AESNI_AVAILABLE + #if defined(__e2k__) || \ + (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800)) + #define T1HA0_AESNI_AVAILABLE 1 + #else + #define T1HA0_AESNI_AVAILABLE 0 + #endif + #endif /* ifndef T1HA0_AESNI_AVAILABLE */ + + #if T1HA0_AESNI_AVAILABLE +T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_noavx(void); +T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx(void); + #ifndef __e2k__ +T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx2(void); + #endif + #endif /* if T1HA0_AESNI_AVAILABLE */ +#endif /* T1HA0_DISABLED */ + +/****************************************************************************** + * + * t1ha2 = 64 and 128-bit, SLIGHTLY MORE ATTENTION FOR QUALITY AND STRENGTH. + * + * - The recommended version of "Fast Positive Hash" with good quality + * for checksum, hash tables and fingerprinting. + * - Portable and extremely efficiency on modern 64-bit CPUs. + * Designed for 64-bit little-endian platforms, + * in other cases will runs slowly. + * - Great quality of hashing and still faster than other non-t1ha hashes. + * Provides streaming mode and 128-bit result. + * + * Note: Due performance reason 64- and 128-bit results are completely + * different each other, i.e. 64-bit result is NOT any part of 128-bit. + */ +#ifndef T1HA2_DISABLED + +/* The at-once variant with 64-bit result */ +T1HA_API uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed); + +/* The at-once variant with 128-bit result. + * Argument `extra_result` is NOT optional and MUST be valid. + * The high 64-bit part of 128-bit hash will be always unconditionally + * stored to the address given by `extra_result` argument. */ +T1HA_API uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result, + const void *__restrict data, size_t length, + uint64_t seed); + +/* The init/update/final trinity for streaming. + * Return 64 or 128-bit result depentently from `extra_result` argument. */ +T1HA_API void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y); +T1HA_API void t1ha2_update(t1ha_context_t *__restrict ctx, + const void *__restrict data, size_t length); + +/* Argument `extra_result` is optional and MAY be NULL. + * - If `extra_result` is NOT NULL then the 128-bit hash will be calculated, + * and high 64-bit part of it will be stored to the address given + * by `extra_result` argument. + * - Otherwise the 64-bit hash will be calculated + * and returned from function directly. + * + * Note: Due performance reason 64- and 128-bit results are completely + * different each other, i.e. 64-bit result is NOT any part of 128-bit. */ +T1HA_API uint64_t t1ha2_final(t1ha_context_t *__restrict ctx, + uint64_t *__restrict extra_result /* optional */); + +#endif /* T1HA2_DISABLED */ + +/****************************************************************************** + * + * t1ha1 = 64-bit, BASELINE FAST PORTABLE HASH: + * + * - Runs faster on 64-bit platforms in other cases may runs slowly. + * - Portable and stable, returns same 64-bit result + * on all architectures and CPUs. + * - Unfortunately it fails the "strict avalanche criteria", + * see test results at https://github.com/demerphq/smhasher. + * + * This flaw is insignificant for the t1ha1() purposes and imperceptible + * from a practical point of view. + * However, nowadays this issue has resolved in the next t1ha2(), + * that was initially planned to providing a bit more quality. + */ +#ifndef T1HA1_DISABLED + +/* The little-endian variant. */ +T1HA_API uint64_t t1ha1_le(const void *data, size_t length, uint64_t seed); + +/* The big-endian variant. */ +T1HA_API uint64_t t1ha1_be(const void *data, size_t length, uint64_t seed); + +#endif /* T1HA1_DISABLED */ + +/****************************************************************************** + * + * t1ha0 = 64-bit, JUST ONLY FASTER: + * + * - Provides fast-as-possible hashing for current CPU, including + * 32-bit systems and engaging the available hardware acceleration. + * - It is a facade that selects most quick-and-dirty hash + * for the current processor. For instance, on IA32 (x86) actual function + * will be selected in runtime, depending on current CPU capabilities + * + * BE CAREFUL!!! THIS IS MEANS: + * + * 1. The quality of hash is a subject for tradeoffs with performance. + * So, the quality and strength of t1ha0() may be lower than t1ha1(), + * especially on 32-bit targets, but then much faster. + * However, guaranteed that it passes all SMHasher tests. + * + * 2. No warranty that the hash result will be same for particular + * key on another machine or another version of libt1ha. + * + * Briefly, such hash-results and their derivatives, should be + * used only in runtime, but should not be persist or transferred + * over a network. + * + * + * When T1HA0_RUNTIME_SELECT is nonzero the t1ha0_resolve() function could + * be used to get actual t1ha0() implementation address at runtime. This is + * useful for two cases: + * - calling by local pointer-to-function usually is little + * bit faster (less overhead) than via a PLT thru the DSO boundary. + * - GNU Indirect functions (see below) don't supported by environment + * and calling by t1ha0_funcptr is not available and/or expensive. + */ + +#ifndef T1HA0_DISABLED + +/* The little-endian variant for 32-bit CPU. */ +uint64_t t1ha0_32le(const void *data, size_t length, uint64_t seed); +/* The big-endian variant for 32-bit CPU. */ +uint64_t t1ha0_32be(const void *data, size_t length, uint64_t seed); + + /* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */ + #ifndef T1HA0_AESNI_AVAILABLE + #if defined(__e2k__) || \ + (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800)) + #define T1HA0_AESNI_AVAILABLE 1 + #else + #define T1HA0_AESNI_AVAILABLE 0 + #endif + #endif /* T1HA0_AESNI_AVAILABLE */ + + /* Define T1HA0_RUNTIME_SELECT to 0 for disable dispatching t1ha0 at runtime. + */ + #ifndef T1HA0_RUNTIME_SELECT + #if T1HA0_AESNI_AVAILABLE && !defined(__e2k__) + #define T1HA0_RUNTIME_SELECT 1 + #else + #define T1HA0_RUNTIME_SELECT 0 + #endif + #endif /* T1HA0_RUNTIME_SELECT */ + + #if !T1HA0_RUNTIME_SELECT && !defined(T1HA0_USE_DEFINE) + #if defined(__LCC__) + #define T1HA0_USE_DEFINE 1 + #else + #define T1HA0_USE_DEFINE 0 + #endif + #endif /* T1HA0_USE_DEFINE */ + + #if T1HA0_AESNI_AVAILABLE +uint64_t t1ha0_ia32aes_noavx(const void *data, size_t length, uint64_t seed); +uint64_t t1ha0_ia32aes_avx(const void *data, size_t length, uint64_t seed); + #ifndef __e2k__ +uint64_t t1ha0_ia32aes_avx2(const void *data, size_t length, uint64_t seed); + #endif + #endif /* T1HA0_AESNI_AVAILABLE */ + + #if T1HA0_RUNTIME_SELECT +typedef uint64_t (*t1ha0_function_t)(const void *, size_t, uint64_t); +T1HA_API t1ha0_function_t t1ha0_resolve(void); + #if T1HA_USE_INDIRECT_FUNCTIONS +T1HA_API uint64_t t1ha0(const void *data, size_t length, uint64_t seed); + #else +/* Otherwise function pointer will be used. + * Unfortunately this may cause some overhead calling. */ +T1HA_API extern uint64_t (*t1ha0_funcptr)(const void *data, size_t length, + uint64_t seed); +static __force_inline uint64_t t1ha0(const void *data, size_t length, + uint64_t seed) { + + return t1ha0_funcptr(data, length, seed); + +} + + #endif /* T1HA_USE_INDIRECT_FUNCTIONS */ + + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + + #if T1HA0_USE_DEFINE + + #if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) + #if defined(T1HA1_DISABLED) + #define t1ha0 t1ha2_atonce + #else + #define t1ha0 t1ha1_be + #endif /* T1HA1_DISABLED */ + #else /* 32/64 */ + #define t1ha0 t1ha0_32be + #endif /* 32/64 */ + + #else /* T1HA0_USE_DEFINE */ + +static __force_inline uint64_t t1ha0(const void *data, size_t length, + uint64_t seed) { + + #if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) + #if defined(T1HA1_DISABLED) + return t1ha2_atonce(data, length, seed); + #else + return t1ha1_be(data, length, seed); + #endif /* T1HA1_DISABLED */ + #else /* 32/64 */ + return t1ha0_32be(data, length, seed); + #endif /* 32/64 */ + +} + + #endif /* !T1HA0_USE_DEFINE */ + + #else /* !T1HA0_RUNTIME_SELECT && __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ + + #if T1HA0_USE_DEFINE + + #if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) + #if defined(T1HA1_DISABLED) + #define t1ha0 t1ha2_atonce + #else + #define t1ha0 t1ha1_le + #endif /* T1HA1_DISABLED */ + #else /* 32/64 */ + #define t1ha0 t1ha0_32le + #endif /* 32/64 */ + + #else + +static __force_inline uint64_t t1ha0(const void *data, size_t length, + uint64_t seed) { + + #if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) + #if defined(T1HA1_DISABLED) + return t1ha2_atonce(data, length, seed); + #else + return t1ha1_le(data, length, seed); + #endif /* T1HA1_DISABLED */ + #else /* 32/64 */ + return t1ha0_32le(data, length, seed); + #endif /* 32/64 */ + +} + + #endif /* !T1HA0_USE_DEFINE */ + + #endif /* !T1HA0_RUNTIME_SELECT */ + +#endif /* T1HA0_DISABLED */ + +#ifdef __cplusplus + +} + +#endif + +#if __GNUC_PREREQ(4, 0) + #pragma GCC visibility pop +#endif /* __GNUC_PREREQ(4,0) */ + diff --git a/include/t1ha0_ia32aes_b.h b/include/t1ha0_ia32aes_b.h new file mode 100644 index 00000000..93b16771 --- /dev/null +++ b/include/t1ha0_ia32aes_b.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com, + * Fast Positive Hash. + * + * Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>, + * The 1Hippeus project (t1h). + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgement in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * t1ha = { Fast Positive Hash, aka "ŠŠ¾Š·ŠøŃŠøŠ²Š½ŃŠ¹ Š„ŃŃ" } + * by [Positive Technologies](https://www.ptsecurity.ru) + * + * Briefly, it is a 64-bit Hash Function: + * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64, + * but portable and without penalties it can run on any 64-bit CPU. + * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash + * and all others portable hash-functions (which do not use specific + * hardware tricks). + * 3. Not suitable for cryptography. + * + * The Future will (be) Positive. ŠŃŃ Š±ŃŠ“ŠµŃ Ń Š¾ŃŠ¾ŃŠ¾. + * + * ACKNOWLEDGEMENT: + * The t1ha was originally developed by Leonid Yuriev (ŠŠµŠ¾Š½ŠøŠ“ Š®ŃŃŠµŠ²) + * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta! + */ + +#include "t1ha_bits.h" +#include "t1ha_selfcheck.h" + +#if T1HA0_AESNI_AVAILABLE + +uint64_t T1HA_IA32AES_NAME(const void *data, uint32_t len) { + + uint64_t a = 0; + uint64_t b = len; + + if (likely(len > 32)) { + + __m128i x = _mm_set_epi64x(a, b); + __m128i y = _mm_aesenc_si128(x, _mm_set_epi64x(prime_0, prime_1)); + + const __m128i *v = (const __m128i *)data; + const __m128i *const detent = + (const __m128i *)((const uint8_t *)data + (len & ~15ul)); + data = detent; + + if (len & 16) { + + x = _mm_add_epi64(x, _mm_loadu_si128(v++)); + y = _mm_aesenc_si128(x, y); + + } + + len &= 15; + + if (v + 7 < detent) { + + __m128i salt = y; + do { + + __m128i t = _mm_aesenc_si128(_mm_loadu_si128(v++), salt); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + + salt = _mm_add_epi64(salt, _mm_set_epi64x(prime_5, prime_6)); + t = _mm_aesenc_si128(x, t); + x = _mm_add_epi64(y, x); + y = t; + + } while (v + 7 < detent); + + } + + while (v < detent) { + + __m128i v0y = _mm_add_epi64(y, _mm_loadu_si128(v++)); + __m128i v1x = _mm_sub_epi64(x, _mm_loadu_si128(v++)); + x = _mm_aesdec_si128(x, v0y); + y = _mm_aesdec_si128(y, v1x); + + } + + x = _mm_add_epi64(_mm_aesdec_si128(x, _mm_aesenc_si128(y, x)), y); + #if defined(__x86_64__) || defined(_M_X64) + #if defined(__SSE4_1__) || defined(__AVX__) + a = _mm_extract_epi64(x, 0); + b = _mm_extract_epi64(x, 1); + #else + a = _mm_cvtsi128_si64(x); + b = _mm_cvtsi128_si64(_mm_unpackhi_epi64(x, x)); + #endif + #else + #if defined(__SSE4_1__) || defined(__AVX__) + a = (uint32_t)_mm_extract_epi32(x, 0) | (uint64_t)_mm_extract_epi32(x, 1) + << 32; + b = (uint32_t)_mm_extract_epi32(x, 2) | (uint64_t)_mm_extract_epi32(x, 3) + << 32; + #else + a = (uint32_t)_mm_cvtsi128_si32(x); + a |= (uint64_t)_mm_cvtsi128_si32(_mm_shuffle_epi32(x, 1)) << 32; + x = _mm_unpackhi_epi64(x, x); + b = (uint32_t)_mm_cvtsi128_si32(x); + b |= (uint64_t)_mm_cvtsi128_si32(_mm_shuffle_epi32(x, 1)) << 32; + #endif + #endif + #ifdef __AVX__ + _mm256_zeroupper(); + #elif !(defined(_X86_64_) || defined(__x86_64__) || defined(_M_X64) || \ + defined(__e2k__)) + _mm_empty(); + #endif + + } + + const uint64_t *v = (const uint64_t *)data; + switch (len) { + + default: + mixup64(&a, &b, fetch64_le_unaligned(v++), prime_4); + /* fall through */ + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + mixup64(&b, &a, fetch64_le_unaligned(v++), prime_3); + /* fall through */ + case 16: + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + mixup64(&a, &b, fetch64_le_unaligned(v++), prime_2); + /* fall through */ + case 8: + case 7: + case 6: + case 5: + case 4: + case 3: + case 2: + case 1: + mixup64(&b, &a, tail64_le_unaligned(v, len), prime_1); + /* fall through */ + case 0: + return final64(a, b); + + } + +} + +#endif /* T1HA0_AESNI_AVAILABLE */ +#undef T1HA_IA32AES_NAME + diff --git a/include/t1ha_bits.h b/include/t1ha_bits.h new file mode 100644 index 00000000..0b9bbda5 --- /dev/null +++ b/include/t1ha_bits.h @@ -0,0 +1,1423 @@ +/* + * Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com, + * Fast Positive Hash. + * + * Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>, + * The 1Hippeus project (t1h). + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgement in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * t1ha = { Fast Positive Hash, aka "ŠŠ¾Š·ŠøŃŠøŠ²Š½ŃŠ¹ Š„ŃŃ" } + * by [Positive Technologies](https://www.ptsecurity.ru) + * + * Briefly, it is a 64-bit Hash Function: + * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64, + * but portable and without penalties it can run on any 64-bit CPU. + * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash + * and all others portable hash-functions (which do not use specific + * hardware tricks). + * 3. Not suitable for cryptography. + * + * The Future will (be) Positive. ŠŃŃ Š±ŃŠ“ŠµŃ Ń Š¾ŃŠ¾ŃŠ¾. + * + * ACKNOWLEDGEMENT: + * The t1ha was originally developed by Leonid Yuriev (ŠŠµŠ¾Š½ŠøŠ“ Š®ŃŃŠµŠ²) + * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta! + */ + +#pragma once + +#if defined(_MSC_VER) + #pragma warning(disable : 4201) /* nameless struct/union */ + #if _MSC_VER > 1800 + #pragma warning(disable : 4464) /* relative include path contains '..' */ + #endif /* 1800 */ +#endif /* MSVC */ +#include "t1ha.h" + +#ifndef T1HA_USE_FAST_ONESHOT_READ + /* Define it to 1 for little bit faster code. + * Unfortunately this may triggering a false-positive alarms from Valgrind, + * AddressSanitizer and other similar tool. + * So, define it to 0 for calmness if doubt. */ + #define T1HA_USE_FAST_ONESHOT_READ 1 +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +/*****************************************************************************/ + +#include <assert.h> /* for assert() */ +#include <stdbool.h> /* for bool */ +#include <string.h> /* for memcpy() */ + +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ + __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ + #error Unsupported byte order. +#endif + +#define T1HA_UNALIGNED_ACCESS__UNABLE 0 +#define T1HA_UNALIGNED_ACCESS__SLOW 1 +#define T1HA_UNALIGNED_ACCESS__EFFICIENT 2 + +#ifndef T1HA_SYS_UNALIGNED_ACCESS + #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT + #elif defined(__ia32__) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT + #elif defined(__e2k__) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__SLOW + #elif defined(__ARM_FEATURE_UNALIGNED) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT + #else + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__UNABLE + #endif +#endif /* T1HA_SYS_UNALIGNED_ACCESS */ + +#define ALIGNMENT_16 2 +#define ALIGNMENT_32 4 +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul + #define ALIGNMENT_64 8 +#else + #define ALIGNMENT_64 4 +#endif + +#ifndef PAGESIZE + #define PAGESIZE 4096 +#endif /* PAGESIZE */ + +/***************************************************************************/ + +#ifndef __has_builtin + #define __has_builtin(x) (0) +#endif + +#ifndef __has_warning + #define __has_warning(x) (0) +#endif + +#ifndef __has_feature + #define __has_feature(x) (0) +#endif + +#ifndef __has_extension + #define __has_extension(x) (0) +#endif + +#if __has_feature(address_sanitizer) + #define __SANITIZE_ADDRESS__ 1 +#endif + +#ifndef __optimize + #if defined(__clang__) && !__has_attribute(__optimize__) + #define __optimize(ops) + #elif defined(__GNUC__) || __has_attribute(__optimize__) + #define __optimize(ops) __attribute__((__optimize__(ops))) + #else + #define __optimize(ops) + #endif +#endif /* __optimize */ + +#ifndef __cold + #if defined(__OPTIMIZE__) + #if defined(__e2k__) + #define __cold __optimize(1) __attribute__((__cold__)) + #elif defined(__clang__) && !__has_attribute(__cold__) && \ + __has_attribute(__section__) + /* just put infrequently used functions in separate section */ + #define __cold \ + __attribute__((__section__("text.unlikely"))) __optimize("Os") + #elif defined(__GNUC__) || __has_attribute(__cold__) + #define __cold __attribute__((__cold__)) __optimize("Os") + #else + #define __cold __optimize("Os") + #endif + #else + #define __cold + #endif +#endif /* __cold */ + +#if __GNUC_PREREQ(4, 4) || defined(__clang__) + + #if defined(__ia32__) || defined(__e2k__) + #include <x86intrin.h> + #endif + + #if defined(__ia32__) && !defined(__cpuid_count) + #include <cpuid.h> + #endif + + #if defined(__e2k__) + #include <e2kbuiltin.h> + #endif + + #ifndef likely + #define likely(cond) __builtin_expect(!!(cond), 1) + #endif + + #ifndef unlikely + #define unlikely(cond) __builtin_expect(!!(cond), 0) + #endif + + #if __GNUC_PREREQ(4, 5) || __has_builtin(__builtin_unreachable) + #define unreachable() __builtin_unreachable() + #endif + + #define bswap64(v) __builtin_bswap64(v) + #define bswap32(v) __builtin_bswap32(v) + #if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16) + #define bswap16(v) __builtin_bswap16(v) + #endif + + #if !defined(__maybe_unused) && \ + (__GNUC_PREREQ(4, 3) || __has_attribute(__unused__)) + #define __maybe_unused __attribute__((__unused__)) + #endif + + #if !defined(__always_inline) && \ + (__GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__)) + #define __always_inline __inline __attribute__((__always_inline__)) + #endif + + #if defined(__e2k__) + + #if __iset__ >= 3 + #define mul_64x64_high(a, b) __builtin_e2k_umulhd(a, b) + #endif /* __iset__ >= 3 */ + + #if __iset__ >= 5 +static __maybe_unused __always_inline unsigned e2k_add64carry_first( + uint64_t base, uint64_t addend, uint64_t *sum) { + + *sum = base + addend; + return (unsigned)__builtin_e2k_addcd_c(base, addend, 0); + +} + + #define add64carry_first(base, addend, sum) \ + e2k_add64carry_first(base, addend, sum) + +static __maybe_unused __always_inline unsigned e2k_add64carry_next( + unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) { + + *sum = __builtin_e2k_addcd(base, addend, carry); + return (unsigned)__builtin_e2k_addcd_c(base, addend, carry); + +} + + #define add64carry_next(carry, base, addend, sum) \ + e2k_add64carry_next(carry, base, addend, sum) + +static __maybe_unused __always_inline void e2k_add64carry_last(unsigned carry, + uint64_t base, + uint64_t addend, + uint64_t *sum) { + + *sum = __builtin_e2k_addcd(base, addend, carry); + +} + + #define add64carry_last(carry, base, addend, sum) \ + e2k_add64carry_last(carry, base, addend, sum) + #endif /* __iset__ >= 5 */ + + #define fetch64_be_aligned(ptr) ((uint64_t)__builtin_e2k_ld_64s_be(ptr)) + #define fetch32_be_aligned(ptr) ((uint32_t)__builtin_e2k_ld_32u_be(ptr)) + + #endif /* __e2k__ Elbrus */ + +#elif defined(_MSC_VER) + + #if _MSC_FULL_VER < 190024234 && defined(_M_IX86) + #pragma message( \ + "For AES-NI at least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required.") + #endif + #if _MSC_FULL_VER < 191526730 + #pragma message( \ + "It is recommended to use \"Microsoft C/C++ Compiler\" version 19.15.26730 (Visual Studio 2017 15.8) or newer.") + #endif + #if _MSC_FULL_VER < 180040629 + #error At least "Microsoft C/C++ Compiler" version 18.00.40629 (Visual Studio 2013 Update 5) is required. + #endif + + #pragma warning(push, 1) + + #include <intrin.h> + #include <stdlib.h> + #define likely(cond) (cond) + #define unlikely(cond) (cond) + #define unreachable() __assume(0) + #define bswap64(v) _byteswap_uint64(v) + #define bswap32(v) _byteswap_ulong(v) + #define bswap16(v) _byteswap_ushort(v) + #define rot64(v, s) _rotr64(v, s) + #define rot32(v, s) _rotr(v, s) + #define __always_inline __forceinline + + #if defined(_M_X64) || defined(_M_IA64) + #pragma intrinsic(_umul128) + #define mul_64x64_128(a, b, ph) _umul128(a, b, ph) + #pragma intrinsic(_addcarry_u64) + #define add64carry_first(base, addend, sum) \ + _addcarry_u64(0, base, addend, sum) + #define add64carry_next(carry, base, addend, sum) \ + _addcarry_u64(carry, base, addend, sum) + #define add64carry_last(carry, base, addend, sum) \ + (void)_addcarry_u64(carry, base, addend, sum) + #endif + + #if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + #pragma intrinsic(__umulh) + #define mul_64x64_high(a, b) __umulh(a, b) + #endif + + #if defined(_M_IX86) + #pragma intrinsic(__emulu) + #define mul_32x32_64(a, b) __emulu(a, b) + + #if _MSC_VER >= 1915 /* LY: workaround for SSA-optimizer bug */ + #pragma intrinsic(_addcarry_u32) + #define add32carry_first(base, addend, sum) \ + _addcarry_u32(0, base, addend, sum) + #define add32carry_next(carry, base, addend, sum) \ + _addcarry_u32(carry, base, addend, sum) + #define add32carry_last(carry, base, addend, sum) \ + (void)_addcarry_u32(carry, base, addend, sum) + +static __forceinline char msvc32_add64carry_first(uint64_t base, + uint64_t addend, + uint64_t *sum) { + + uint32_t *const sum32 = (uint32_t *)sum; + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); + return add32carry_next(add32carry_first(base_32l, addend_32l, sum32), + base_32h, addend_32h, sum32 + 1); + +} + + #define add64carry_first(base, addend, sum) \ + msvc32_add64carry_first(base, addend, sum) + +static __forceinline char msvc32_add64carry_next(char carry, uint64_t base, + uint64_t addend, + uint64_t *sum) { + + uint32_t *const sum32 = (uint32_t *)sum; + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); + return add32carry_next(add32carry_next(carry, base_32l, addend_32l, sum32), + base_32h, addend_32h, sum32 + 1); + +} + + #define add64carry_next(carry, base, addend, sum) \ + msvc32_add64carry_next(carry, base, addend, sum) + +static __forceinline void msvc32_add64carry_last(char carry, uint64_t base, + uint64_t addend, + uint64_t *sum) { + + uint32_t *const sum32 = (uint32_t *)sum; + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); + add32carry_last(add32carry_next(carry, base_32l, addend_32l, sum32), base_32h, + addend_32h, sum32 + 1); + +} + + #define add64carry_last(carry, base, addend, sum) \ + msvc32_add64carry_last(carry, base, addend, sum) + #endif /* _MSC_FULL_VER >= 190024231 */ + + #elif defined(_M_ARM) + #define mul_32x32_64(a, b) _arm_umull(a, b) + #endif + + #pragma warning(pop) + #pragma warning(disable : 4514) /* 'xyz': unreferenced inline function \ + has been removed */ + #pragma warning(disable : 4710) /* 'xyz': function not inlined */ + #pragma warning(disable : 4711) /* function 'xyz' selected for \ + automatic inline expansion */ + #pragma warning(disable : 4127) /* conditional expression is constant */ + #pragma warning(disable : 4702) /* unreachable code */ +#endif /* Compiler */ + +#ifndef likely + #define likely(cond) (cond) +#endif +#ifndef unlikely + #define unlikely(cond) (cond) +#endif +#ifndef __maybe_unused + #define __maybe_unused +#endif +#ifndef __always_inline + #define __always_inline __inline +#endif +#ifndef unreachable + #define unreachable() \ + do { \ + \ + } while (1) +#endif + +#ifndef bswap64 + #if defined(bswap_64) + #define bswap64 bswap_64 + #elif defined(__bswap_64) + #define bswap64 __bswap_64 + #else +static __always_inline uint64_t bswap64(uint64_t v) { + + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | + ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | + ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | + ((v >> 40) & UINT64_C(0x000000000000ff00)); + +} + + #endif +#endif /* bswap64 */ + +#ifndef bswap32 + #if defined(bswap_32) + #define bswap32 bswap_32 + #elif defined(__bswap_32) + #define bswap32 __bswap_32 + #else +static __always_inline uint32_t bswap32(uint32_t v) { + + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | + ((v >> 8) & UINT32_C(0x0000ff00)); + +} + + #endif +#endif /* bswap32 */ + +#ifndef bswap16 + #if defined(bswap_16) + #define bswap16 bswap_16 + #elif defined(__bswap_16) + #define bswap16 __bswap_16 + #else +static __always_inline uint16_t bswap16(uint16_t v) { + + return v << 8 | v >> 8; + +} + + #endif +#endif /* bswap16 */ + +#if defined(__ia32__) || \ + T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT + /* The __builtin_assume_aligned() leads gcc/clang to load values into the + * registers, even when it is possible to directly use an operand from memory. + * This can lead to a shortage of registers and a significant slowdown. + * Therefore avoid unnecessary use of __builtin_assume_aligned() for x86. */ + #define read_unaligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) + #define read_aligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) +#endif /* __ia32__ */ + +#ifndef read_unaligned + #if defined(__GNUC__) || __has_attribute(__packed__) +typedef struct { + + uint8_t unaligned_8; + uint16_t unaligned_16; + uint32_t unaligned_32; + uint64_t unaligned_64; + +} __attribute__((__packed__)) t1ha_unaligned_proxy; + + #define read_unaligned(ptr, bits) \ + (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ + t1ha_unaligned_proxy, unaligned_##bits))) \ + ->unaligned_##bits) + #elif defined(_MSC_VER) + #pragma warning( \ + disable : 4235) /* nonstandard extension used: '__unaligned' \ + * keyword not supported on this architecture */ + #define read_unaligned(ptr, bits) \ + (*(const __unaligned uint##bits##_t *)(ptr)) + #else + #pragma pack(push, 1) +typedef struct { + + uint8_t unaligned_8; + uint16_t unaligned_16; + uint32_t unaligned_32; + uint64_t unaligned_64; + +} t1ha_unaligned_proxy; + + #pragma pack(pop) + #define read_unaligned(ptr, bits) \ + (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ + t1ha_unaligned_proxy, unaligned_##bits))) \ + ->unaligned_##bits) + #endif +#endif /* read_unaligned */ + +#ifndef read_aligned + #if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned) + #define read_aligned(ptr, bits) \ + (*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits)) + #elif (__GNUC_PREREQ(3, 3) || __has_attribute(__aligned__)) && \ + !defined(__clang__) + #define read_aligned(ptr, bits) \ + (*(const uint##bits##_t \ + __attribute__((__aligned__(ALIGNMENT_##bits))) *)(ptr)) + #elif __has_attribute(__assume_aligned__) + +static __always_inline const uint16_t *__attribute__(( + __assume_aligned__(ALIGNMENT_16))) cast_aligned_16(const void *ptr) { + + return (const uint16_t *)ptr; + +} + +static __always_inline const uint32_t *__attribute__(( + __assume_aligned__(ALIGNMENT_32))) cast_aligned_32(const void *ptr) { + + return (const uint32_t *)ptr; + +} + +static __always_inline const uint64_t *__attribute__(( + __assume_aligned__(ALIGNMENT_64))) cast_aligned_64(const void *ptr) { + + return (const uint64_t *)ptr; + +} + + #define read_aligned(ptr, bits) (*cast_aligned_##bits(ptr)) + + #elif defined(_MSC_VER) + #define read_aligned(ptr, bits) \ + (*(const __declspec(align(ALIGNMENT_##bits)) uint##bits##_t *)(ptr)) + #else + #define read_aligned(ptr, bits) (*(const uint##bits##_t *)(ptr)) + #endif +#endif /* read_aligned */ + +#ifndef prefetch + #if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \ + !defined(__ia32__) + #define prefetch(ptr) __builtin_prefetch(ptr) + #elif defined(_M_ARM64) || defined(_M_ARM) + #define prefetch(ptr) __prefetch(ptr) + #else + #define prefetch(ptr) \ + do { \ + \ + (void)(ptr); \ + \ + } while (0) + + #endif +#endif /* prefetch */ + +#if __has_warning("-Wconstant-logical-operand") + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wconstant-logical-operand" + #elif defined(__GNUC__) + #pragma GCC diagnostic ignored "-Wconstant-logical-operand" + #else + #pragma warning disable "constant-logical-operand" + #endif +#endif /* -Wconstant-logical-operand */ + +#if __has_warning("-Wtautological-pointer-compare") + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wtautological-pointer-compare" + #elif defined(__GNUC__) + #pragma GCC diagnostic ignored "-Wtautological-pointer-compare" + #else + #pragma warning disable "tautological-pointer-compare" + #endif +#endif /* -Wtautological-pointer-compare */ + +/***************************************************************************/ + +#if __GNUC_PREREQ(4, 0) + #pragma GCC visibility push(hidden) +#endif /* __GNUC_PREREQ(4,0) */ + +/*---------------------------------------------------------- Little Endian */ + +#ifndef fetch16_le_aligned +static __maybe_unused __always_inline uint16_t +fetch16_le_aligned(const void *v) { + + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_aligned(v, 16); + #else + return bswap16(read_aligned(v, 16)); + #endif + +} + +#endif /* fetch16_le_aligned */ + +#ifndef fetch16_le_unaligned +static __maybe_unused __always_inline uint16_t +fetch16_le_unaligned(const void *v) { + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + const uint8_t *p = (const uint8_t *)v; + return p[0] | (uint16_t)p[1] << 8; + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_unaligned(v, 16); + #else + return bswap16(read_unaligned(v, 16)); + #endif + +} + +#endif /* fetch16_le_unaligned */ + +#ifndef fetch32_le_aligned +static __maybe_unused __always_inline uint32_t +fetch32_le_aligned(const void *v) { + + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_aligned(v, 32); + #else + return bswap32(read_aligned(v, 32)); + #endif + +} + +#endif /* fetch32_le_aligned */ + +#ifndef fetch32_le_unaligned +static __maybe_unused __always_inline uint32_t +fetch32_le_unaligned(const void *v) { + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + return fetch16_le_unaligned(v) | + (uint32_t)fetch16_le_unaligned((const uint8_t *)v + 2) << 16; + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_unaligned(v, 32); + #else + return bswap32(read_unaligned(v, 32)); + #endif + +} + +#endif /* fetch32_le_unaligned */ + +#ifndef fetch64_le_aligned +static __maybe_unused __always_inline uint64_t +fetch64_le_aligned(const void *v) { + + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_aligned(v, 64); + #else + return bswap64(read_aligned(v, 64)); + #endif + +} + +#endif /* fetch64_le_aligned */ + +#ifndef fetch64_le_unaligned +static __maybe_unused __always_inline uint64_t +fetch64_le_unaligned(const void *v) { + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + return fetch32_le_unaligned(v) | + (uint64_t)fetch32_le_unaligned((const uint8_t *)v + 4) << 32; + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_unaligned(v, 64); + #else + return bswap64(read_unaligned(v, 64)); + #endif + +} + +#endif /* fetch64_le_unaligned */ + +static __maybe_unused __always_inline uint64_t tail64_le_aligned(const void *v, + size_t tail) { + + const uint8_t *const p = (const uint8_t *)v; +#if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__) + /* We can perform a 'oneshot' read, which is little bit faster. */ + const unsigned shift = ((8 - tail) & 7) << 3; + return fetch64_le_aligned(p) & ((~UINT64_C(0)) >> shift); +#else + uint64_t r = 0; + switch (tail & 7) { + + default: + unreachable(); + /* fall through */ + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* For most CPUs this code is better when not needed byte reordering. */ + case 0: + return fetch64_le_aligned(p); + case 7: + r = (uint64_t)p[6] << 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 32; + /* fall through */ + case 4: + return r + fetch32_le_aligned(p); + case 3: + r = (uint64_t)p[2] << 16; + /* fall through */ + case 2: + return r + fetch16_le_aligned(p); + case 1: + return p[0]; + #else + case 0: + r = p[7] << 8; + /* fall through */ + case 7: + r += p[6]; + r <<= 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 8; + /* fall through */ + case 4: + r += p[3]; + r <<= 8; + /* fall through */ + case 3: + r += p[2]; + r <<= 8; + /* fall through */ + case 2: + r += p[1]; + r <<= 8; + /* fall through */ + case 1: + return r + p[0]; + #endif + + } + +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +} + +#if T1HA_USE_FAST_ONESHOT_READ && \ + T1HA_SYS_UNALIGNED_ACCESS != T1HA_UNALIGNED_ACCESS__UNABLE && \ + defined(PAGESIZE) && PAGESIZE > 42 && !defined(__SANITIZE_ADDRESS__) + #define can_read_underside(ptr, size) \ + (((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0) +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +static __maybe_unused __always_inline uint64_t +tail64_le_unaligned(const void *v, size_t tail) { + + const uint8_t *p = (const uint8_t *)v; +#if defined(can_read_underside) && \ + (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) + /* On some systems (e.g. x86_64) we can perform a 'oneshot' read, which + * is little bit faster. Thanks Marcin Å»ukowski <marcin.zukowski@gmail.com> + * for the reminder. */ + const unsigned offset = (8 - tail) & 7; + const unsigned shift = offset << 3; + if (likely(can_read_underside(p, 8))) { + + p -= offset; + return fetch64_le_unaligned(p) >> shift; + + } + + return fetch64_le_unaligned(p) & ((~UINT64_C(0)) >> shift); +#else + uint64_t r = 0; + switch (tail & 7) { + + default: + unreachable(); + /* fall through */ + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* For most CPUs this code is better when not needed + * copying for alignment or byte reordering. */ + case 0: + return fetch64_le_unaligned(p); + case 7: + r = (uint64_t)p[6] << 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 32; + /* fall through */ + case 4: + return r + fetch32_le_unaligned(p); + case 3: + r = (uint64_t)p[2] << 16; + /* fall through */ + case 2: + return r + fetch16_le_unaligned(p); + case 1: + return p[0]; + #else + /* For most CPUs this code is better than a + * copying for alignment and/or byte reordering. */ + case 0: + r = p[7] << 8; + /* fall through */ + case 7: + r += p[6]; + r <<= 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 8; + /* fall through */ + case 4: + r += p[3]; + r <<= 8; + /* fall through */ + case 3: + r += p[2]; + r <<= 8; + /* fall through */ + case 2: + r += p[1]; + r <<= 8; + /* fall through */ + case 1: + return r + p[0]; + #endif + + } + +#endif /* can_read_underside */ + +} + +/*------------------------------------------------------------- Big Endian */ + +#ifndef fetch16_be_aligned +static __maybe_unused __always_inline uint16_t +fetch16_be_aligned(const void *v) { + + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_aligned(v, 16); + #else + return bswap16(read_aligned(v, 16)); + #endif + +} + +#endif /* fetch16_be_aligned */ + +#ifndef fetch16_be_unaligned +static __maybe_unused __always_inline uint16_t +fetch16_be_unaligned(const void *v) { + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + const uint8_t *p = (const uint8_t *)v; + return (uint16_t)p[0] << 8 | p[1]; + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_unaligned(v, 16); + #else + return bswap16(read_unaligned(v, 16)); + #endif + +} + +#endif /* fetch16_be_unaligned */ + +#ifndef fetch32_be_aligned +static __maybe_unused __always_inline uint32_t +fetch32_be_aligned(const void *v) { + + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_aligned(v, 32); + #else + return bswap32(read_aligned(v, 32)); + #endif + +} + +#endif /* fetch32_be_aligned */ + +#ifndef fetch32_be_unaligned +static __maybe_unused __always_inline uint32_t +fetch32_be_unaligned(const void *v) { + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + return (uint32_t)fetch16_be_unaligned(v) << 16 | + fetch16_be_unaligned((const uint8_t *)v + 2); + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_unaligned(v, 32); + #else + return bswap32(read_unaligned(v, 32)); + #endif + +} + +#endif /* fetch32_be_unaligned */ + +#ifndef fetch64_be_aligned +static __maybe_unused __always_inline uint64_t +fetch64_be_aligned(const void *v) { + + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_aligned(v, 64); + #else + return bswap64(read_aligned(v, 64)); + #endif + +} + +#endif /* fetch64_be_aligned */ + +#ifndef fetch64_be_unaligned +static __maybe_unused __always_inline uint64_t +fetch64_be_unaligned(const void *v) { + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + return (uint64_t)fetch32_be_unaligned(v) << 32 | + fetch32_be_unaligned((const uint8_t *)v + 4); + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_unaligned(v, 64); + #else + return bswap64(read_unaligned(v, 64)); + #endif + +} + +#endif /* fetch64_be_unaligned */ + +static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, + size_t tail) { + + const uint8_t *const p = (const uint8_t *)v; +#if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__) + /* We can perform a 'oneshot' read, which is little bit faster. */ + const unsigned shift = ((8 - tail) & 7) << 3; + return fetch64_be_aligned(p) >> shift; +#else + switch (tail & 7) { + + default: + unreachable(); + /* fall through */ + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + /* For most CPUs this code is better when not byte reordering. */ + case 1: + return p[0]; + case 2: + return fetch16_be_aligned(p); + case 3: + return (uint32_t)fetch16_be_aligned(p) << 8 | p[2]; + case 4: + return fetch32_be_aligned(p); + case 5: + return (uint64_t)fetch32_be_aligned(p) << 8 | p[4]; + case 6: + return (uint64_t)fetch32_be_aligned(p) << 16 | fetch16_be_aligned(p + 4); + case 7: + return (uint64_t)fetch32_be_aligned(p) << 24 | + (uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6]; + case 0: + return fetch64_be_aligned(p); + #else + case 1: + return p[0]; + case 2: + return p[1] | (uint32_t)p[0] << 8; + case 3: + return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; + case 4: + return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | + (uint32_t)p[0] << 24; + case 5: + return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | + (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; + case 6: + return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | + (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; + case 7: + return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | + (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | + (uint64_t)p[1] << 40 | (uint64_t)p[0] << 48; + case 0: + return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | + (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | + (uint64_t)p[2] << 40 | (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; + #endif + + } + +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +} + +static __maybe_unused __always_inline uint64_t +tail64_be_unaligned(const void *v, size_t tail) { + + const uint8_t *p = (const uint8_t *)v; +#if defined(can_read_underside) && \ + (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) + /* On some systems (e.g. x86_64) we can perform a 'oneshot' read, which + * is little bit faster. Thanks Marcin Å»ukowski <marcin.zukowski@gmail.com> + * for the reminder. */ + const unsigned offset = (8 - tail) & 7; + const unsigned shift = offset << 3; + if (likely(can_read_underside(p, 8))) { + + p -= offset; + return fetch64_be_unaligned(p) & ((~UINT64_C(0)) >> shift); + + } + + return fetch64_be_unaligned(p) >> shift; +#else + switch (tail & 7) { + + default: + unreachable(); + /* fall through */ + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + /* For most CPUs this code is better when not needed + * copying for alignment or byte reordering. */ + case 1: + return p[0]; + case 2: + return fetch16_be_unaligned(p); + case 3: + return (uint32_t)fetch16_be_unaligned(p) << 8 | p[2]; + case 4: + return fetch32_be(p); + case 5: + return (uint64_t)fetch32_be_unaligned(p) << 8 | p[4]; + case 6: + return (uint64_t)fetch32_be_unaligned(p) << 16 | + fetch16_be_unaligned(p + 4); + case 7: + return (uint64_t)fetch32_be_unaligned(p) << 24 | + (uint32_t)fetch16_be_unaligned(p + 4) << 8 | p[6]; + case 0: + return fetch64_be_unaligned(p); + #else + /* For most CPUs this code is better than a + * copying for alignment and/or byte reordering. */ + case 1: + return p[0]; + case 2: + return p[1] | (uint32_t)p[0] << 8; + case 3: + return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; + case 4: + return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | + (uint32_t)p[0] << 24; + case 5: + return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | + (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; + case 6: + return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | + (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; + case 7: + return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | + (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | + (uint64_t)p[1] << 40 | (uint64_t)p[0] << 48; + case 0: + return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | + (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | + (uint64_t)p[2] << 40 | (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; + #endif + + } + +#endif /* can_read_underside */ + +} + +/***************************************************************************/ + +#ifndef rot64 +static __maybe_unused __always_inline uint64_t rot64(uint64_t v, unsigned s) { + + return (v >> s) | (v << (64 - s)); + +} + +#endif /* rot64 */ + +#ifndef mul_32x32_64 +static __maybe_unused __always_inline uint64_t mul_32x32_64(uint32_t a, + uint32_t b) { + + return a * (uint64_t)b; + +} + +#endif /* mul_32x32_64 */ + +#ifndef add64carry_first +static __maybe_unused __always_inline unsigned add64carry_first(uint64_t base, + uint64_t addend, + uint64_t *sum) { + + #if __has_builtin(__builtin_addcll) + unsigned long long carryout; + *sum = __builtin_addcll(base, addend, 0, &carryout); + return (unsigned)carryout; + #else + *sum = base + addend; + return *sum < addend; + #endif /* __has_builtin(__builtin_addcll) */ + +} + +#endif /* add64carry_fist */ + +#ifndef add64carry_next +static __maybe_unused __always_inline unsigned add64carry_next(unsigned carry, + uint64_t base, + uint64_t addend, + uint64_t *sum) { + + #if __has_builtin(__builtin_addcll) + unsigned long long carryout; + *sum = __builtin_addcll(base, addend, carry, &carryout); + return (unsigned)carryout; + #else + *sum = base + addend + carry; + return *sum < addend || (carry && *sum == addend); + #endif /* __has_builtin(__builtin_addcll) */ + +} + +#endif /* add64carry_next */ + +#ifndef add64carry_last +static __maybe_unused __always_inline void add64carry_last(unsigned carry, + uint64_t base, + uint64_t addend, + uint64_t *sum) { + + #if __has_builtin(__builtin_addcll) + unsigned long long carryout; + *sum = __builtin_addcll(base, addend, carry, &carryout); + (void)carryout; + #else + *sum = base + addend + carry; + #endif /* __has_builtin(__builtin_addcll) */ + +} + +#endif /* add64carry_last */ + +#ifndef mul_64x64_128 +static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a, + uint64_t b, + uint64_t *h) { + + #if (defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) && \ + (!defined(__LCC__) || __LCC__ != 124) + __uint128_t r = (__uint128_t)a * (__uint128_t)b; + /* modern GCC could nicely optimize this */ + *h = (uint64_t)(r >> 64); + return (uint64_t)r; + #elif defined(mul_64x64_high) + *h = mul_64x64_high(a, b); + return a * b; + #else + /* performs 64x64 to 128 bit multiplication */ + const uint64_t ll = mul_32x32_64((uint32_t)a, (uint32_t)b); + const uint64_t lh = mul_32x32_64(a >> 32, (uint32_t)b); + const uint64_t hl = mul_32x32_64((uint32_t)a, b >> 32); + const uint64_t hh = mul_32x32_64(a >> 32, b >> 32); + + /* Few simplification are possible here for 32-bit architectures, + * but thus we would lost compatibility with the original 64-bit + * version. Think is very bad idea, because then 32-bit t1ha will + * still (relatively) very slowly and well yet not compatible. */ + uint64_t l; + add64carry_last(add64carry_first(ll, lh << 32, &l), hh, lh >> 32, h); + add64carry_last(add64carry_first(l, hl << 32, &l), *h, hl >> 32, h); + return l; + #endif + +} + +#endif /* mul_64x64_128() */ + +#ifndef mul_64x64_high +static __maybe_unused __always_inline uint64_t mul_64x64_high(uint64_t a, + uint64_t b) { + + uint64_t h; + mul_64x64_128(a, b, &h); + return h; + +} + +#endif /* mul_64x64_high */ + +/***************************************************************************/ + +/* 'magic' primes */ +static const uint64_t prime_0 = UINT64_C(0xEC99BF0D8372CAAB); +static const uint64_t prime_1 = UINT64_C(0x82434FE90EDCEF39); +static const uint64_t prime_2 = UINT64_C(0xD4F06DB99D67BE4B); +static const uint64_t prime_3 = UINT64_C(0xBD9CACC22C6E9571); +static const uint64_t prime_4 = UINT64_C(0x9C06FAF4D023E3AB); +static const uint64_t prime_5 = UINT64_C(0xC060724A8424F345); +static const uint64_t prime_6 = UINT64_C(0xCB5AF53AE3AAAC31); + +/* xor high and low parts of full 128-bit product */ +static __maybe_unused __always_inline uint64_t mux64(uint64_t v, + uint64_t prime) { + + uint64_t l, h; + l = mul_64x64_128(v, prime, &h); + return l ^ h; + +} + +static __maybe_unused __always_inline uint64_t final64(uint64_t a, uint64_t b) { + + uint64_t x = (a + rot64(b, 41)) * prime_0; + uint64_t y = (rot64(a, 23) + b) * prime_6; + return mux64(x ^ y, prime_5); + +} + +static __maybe_unused __always_inline void mixup64(uint64_t *__restrict a, + uint64_t *__restrict b, + uint64_t v, uint64_t prime) { + + uint64_t h; + *a ^= mul_64x64_128(*b + v, prime, &h); + *b += h; + +} + +/***************************************************************************/ + +typedef union t1ha_uint128 { + +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + __uint128_t v; +#endif + struct { + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + uint64_t l, h; +#else + uint64_t h, l; +#endif + + }; + +} t1ha_uint128_t; + +static __maybe_unused __always_inline t1ha_uint128_t +not128(const t1ha_uint128_t v) { + + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = ~v.v; +#else + r.l = ~v.l; + r.h = ~v.h; +#endif + return r; + +} + +static __maybe_unused __always_inline t1ha_uint128_t +left128(const t1ha_uint128_t v, unsigned s) { + + t1ha_uint128_t r; + assert(s < 128); +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = v.v << s; +#else + r.l = (s < 64) ? v.l << s : 0; + r.h = (s < 64) ? (v.h << s) | (s ? v.l >> (64 - s) : 0) : v.l << (s - 64); +#endif + return r; + +} + +static __maybe_unused __always_inline t1ha_uint128_t +right128(const t1ha_uint128_t v, unsigned s) { + + t1ha_uint128_t r; + assert(s < 128); +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = v.v >> s; +#else + r.l = (s < 64) ? (s ? v.h << (64 - s) : 0) | (v.l >> s) : v.h >> (s - 64); + r.h = (s < 64) ? v.h >> s : 0; +#endif + return r; + +} + +static __maybe_unused __always_inline t1ha_uint128_t or128(t1ha_uint128_t x, + t1ha_uint128_t y) { + + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = x.v | y.v; +#else + r.l = x.l | y.l; + r.h = x.h | y.h; +#endif + return r; + +} + +static __maybe_unused __always_inline t1ha_uint128_t xor128(t1ha_uint128_t x, + t1ha_uint128_t y) { + + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = x.v ^ y.v; +#else + r.l = x.l ^ y.l; + r.h = x.h ^ y.h; +#endif + return r; + +} + +static __maybe_unused __always_inline t1ha_uint128_t rot128(t1ha_uint128_t v, + unsigned s) { + + s &= 127; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + v.v = (v.v << (128 - s)) | (v.v >> s); + return v; +#else + return s ? or128(left128(v, 128 - s), right128(v, s)) : v; +#endif + +} + +static __maybe_unused __always_inline t1ha_uint128_t add128(t1ha_uint128_t x, + t1ha_uint128_t y) { + + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = x.v + y.v; +#else + add64carry_last(add64carry_first(x.l, y.l, &r.l), x.h, y.h, &r.h); +#endif + return r; + +} + +static __maybe_unused __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x, + t1ha_uint128_t y) { + + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = x.v * y.v; +#else + r.l = mul_64x64_128(x.l, y.l, &r.h); + r.h += x.l * y.h + y.l * x.h; +#endif + return r; + +} + +/***************************************************************************/ + +#if T1HA0_AESNI_AVAILABLE && defined(__ia32__) +uint64_t t1ha_ia32cpu_features(void); + +static __maybe_unused __always_inline bool t1ha_ia32_AESNI_avail( + uint64_t ia32cpu_features) { + + /* check for AES-NI */ + return (ia32cpu_features & UINT32_C(0x02000000)) != 0; + +} + +static __maybe_unused __always_inline bool t1ha_ia32_AVX_avail( + uint64_t ia32cpu_features) { + + /* check for any AVX */ + return (ia32cpu_features & UINT32_C(0x1A000000)) == UINT32_C(0x1A000000); + +} + +static __maybe_unused __always_inline bool t1ha_ia32_AVX2_avail( + uint64_t ia32cpu_features) { + + /* check for 'Advanced Vector Extensions 2' */ + return ((ia32cpu_features >> 32) & 32) != 0; + +} + +#endif /* T1HA0_AESNI_AVAILABLE && __ia32__ */ + diff --git a/include/t1ha_selfcheck.h b/include/t1ha_selfcheck.h new file mode 100644 index 00000000..65343bfe --- /dev/null +++ b/include/t1ha_selfcheck.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com, + * Fast Positive Hash. + * + * Portions Copyright (c) 2010-2020 Leonid Yuriev <leo@yuriev.ru>, + * The 1Hippeus project (t1h). + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgement in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * t1ha = { Fast Positive Hash, aka "ŠŠ¾Š·ŠøŃŠøŠ²Š½ŃŠ¹ Š„ŃŃ" } + * by [Positive Technologies](https://www.ptsecurity.ru) + * + * Briefly, it is a 64-bit Hash Function: + * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64, + * but portable and without penalties it can run on any 64-bit CPU. + * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash + * and all others portable hash-functions (which do not use specific + * hardware tricks). + * 3. Not suitable for cryptography. + * + * The Future will (be) Positive. ŠŃŃ Š±ŃŠ“ŠµŃ Ń Š¾ŃŠ¾ŃŠ¾. + * + * ACKNOWLEDGEMENT: + * The t1ha was originally developed by Leonid Yuriev (ŠŠµŠ¾Š½ŠøŠ“ Š®ŃŃŠµŠ²) + * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta! + */ + +#pragma once +#if defined(_MSC_VER) && _MSC_VER > 1800 + #pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif /* MSVC */ +#include "t1ha.h" + +/***************************************************************************/ +/* Self-checking */ + +extern const uint8_t t1ha_test_pattern[64]; +int t1ha_selfcheck(uint64_t (*hash)(const void *, size_t, uint64_t), + const uint64_t *reference_values); + +#ifndef T1HA2_DISABLED +extern const uint64_t t1ha_refval_2atonce[81]; +extern const uint64_t t1ha_refval_2atonce128[81]; +extern const uint64_t t1ha_refval_2stream[81]; +extern const uint64_t t1ha_refval_2stream128[81]; +#endif /* T1HA2_DISABLED */ + +#ifndef T1HA1_DISABLED +extern const uint64_t t1ha_refval_64le[81]; +extern const uint64_t t1ha_refval_64be[81]; +#endif /* T1HA1_DISABLED */ + +#ifndef T1HA0_DISABLED +extern const uint64_t t1ha_refval_32le[81]; +extern const uint64_t t1ha_refval_32be[81]; + #if T1HA0_AESNI_AVAILABLE +extern const uint64_t t1ha_refval_ia32aes_a[81]; +extern const uint64_t t1ha_refval_ia32aes_b[81]; + #endif /* T1HA0_AESNI_AVAILABLE */ +#endif /* T1HA0_DISABLED */ + diff --git a/include/types.h b/include/types.h index d6476d82..cfb2f3d5 100644 --- a/include/types.h +++ b/include/types.h @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de>, + Heiko Eissfeldt <heiko.eissfeldt@hexco.de>, Andrea Fioraldi <andreafioraldi@gmail.com>, Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -49,6 +49,14 @@ typedef uint128_t u128; #define FS_ERROR_OLD_CMPLOG 32 #define FS_ERROR_OLD_CMPLOG_QEMU 64 +/* New Forkserver */ +#define FS_NEW_VERSION_MIN 1 +#define FS_NEW_VERSION_MAX 1 +#define FS_NEW_ERROR 0xeffe0000 +#define FS_NEW_OPT_MAPSIZE 0x00000001 // parameter: 32 bit value +#define FS_NEW_OPT_SHDMEM_FUZZ 0x00000002 // parameter: none +#define FS_NEW_OPT_AUTODICT 0x00000800 // autodictionary data + /* Reporting options */ #define FS_OPT_ENABLED 0x80000001 #define FS_OPT_MAPSIZE 0x40000000 diff --git a/include/xxhash.h b/include/xxhash.h index 7bc0a14e..991a8f1e 100644 --- a/include/xxhash.h +++ b/include/xxhash.h @@ -32,50 +32,226 @@ * - xxHash homepage: https://www.xxhash.com * - xxHash source repository: https://github.com/Cyan4973/xxHash */ + /*! * @mainpage xxHash * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM + * speed limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 + * flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data + * Velocity | | -------------------- | ------- | ----: | ---------------: | + * ------------------: | | XXH3_64bits() | @b AVX2 | 64 | 59.4 + * GB/s | 133.1 | | MeowHash | AES-NI | 128 | 58.2 + * GB/s | 52.5 | | XXH3_128bits() | @b AVX2 | 128 | 57.9 + * GB/s | 118.1 | | CLHash | PCLMUL | 64 | 37.1 + * GB/s | 58.1 | | XXH3_64bits() | @b SSE2 | 64 | 31.5 + * GB/s | 133.1 | | XXH3_128bits() | @b SSE2 | 128 | 29.6 + * GB/s | 118.1 | | RAM sequential read | | N/A | 28.0 + * GB/s | N/A | | ahash | AES-NI | 64 | 22.5 + * GB/s | 107.2 | | City64 | | 64 | 22.0 + * GB/s | 76.6 | | T1ha2 | | 64 | 22.0 + * GB/s | 99.0 | | City128 | | 128 | 21.7 + * GB/s | 57.7 | | FarmHash | AES-NI | 64 | 21.3 + * GB/s | 71.9 | | XXH64() | | 64 | 19.4 + * GB/s | 71.0 | | SpookyHash | | 64 | 19.3 + * GB/s | 53.2 | | Mum | | 64 | 18.0 + * GB/s | 67.0 | | CRC32C | SSE4.2 | 32 | 13.0 + * GB/s | 57.9 | | XXH32() | | 32 | 9.7 + * GB/s | 71.9 | | City32 | | 32 | 9.1 + * GB/s | 66.0 | | Blake3* | @b AVX2 | 256 | 4.4 + * GB/s | 8.1 | | Murmur3 | | 32 | 3.9 + * GB/s | 56.1 | | SipHash* | | 64 | 3.0 + * GB/s | 43.2 | | Blake3* | @b SSE2 | 256 | 2.4 + * GB/s | 8.1 | | HighwayHash | | 64 | 1.4 + * GB/s | 6.0 | | FNV64 | | 64 | 1.2 + * GB/s | 62.7 | | Blake2* | | 256 | 1.1 + * GB/s | 5.1 | | SHA1* | | 160 | 0.8 + * GB/s | 5.6 | | MD5* | | 128 | 0.6 + * GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also + * noted, even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is + * non-cryptographic by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for + * small data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned + * char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of + * memory, immediately returning the result. They are the easiest and usually + * the fastest option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include <string.h> + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with + * XXH32(). XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include <stdio.h> + * #include <assert.h> + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with + * XXH3_64bits(). XXH64_hash_t hashFile(FILE* f) + * { + + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as + * necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte + * level, since little and big endian conventions will store the same number + * differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits + * first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include <stdio.h> + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable + * format void printXxh32(XXH32_hash_t hash) + * { + + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * * @file xxhash.h * xxHash prototypes and implementation */ -/* TODO: update */ -/* Notice extracted from xxHash homepage: - -xxHash is an extremely fast hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo -@3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MurmurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -Note: SMHasher's CRC32 implementation is not the fastest one. -Other speed-oriented implementations can be faster, -especially in combination with PCLMUL instruction: -https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 - -A 64-bit version, named XXH64, is available since r35. -It offers much better speed, but for 64-bit applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ #if defined(__cplusplus) extern "C" { @@ -86,21 +262,82 @@ extern "C" { * INLINE mode ******************************/ /*! - * XXH_INLINE_ALL (and XXH_PRIVATE_API) - * Use these build macros to inline xxhash into the target unit. - * Inlining improves performance on small inputs, especially when the length is - * expressed as a compile-time constant: - * - * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html - * - * It also keeps xxHash symbols private to the unit, so they are not exported. - * - * Usage: - * #define XXH_INLINE_ALL - * #include "xxhash.h" - * - * Do not compile and link xxhash.o as a separate object, as it is not useful. + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN + /*! + * @brief Gives access to internal state declaration, required for static + * allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ + #define XXH_STATIC_LINKING_ONLY + /* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + + /*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ + #define XXH_IMPLEMENTATION + /* Do not undef XXH_IMPLEMENTATION for Doxygen */ + + /*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length + * is expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ + #define XXH_INLINE_ALL + #undef XXH_INLINE_ALL + /*! + * @brief Exposes the implementation without marking functions as inline. + */ + #define XXH_PRIVATE_API + #undef XXH_PRIVATE_API + /*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries + * which may also include xxHash, you can use @ref XXH_NAMESPACE to + * automatically prefix any public symbol from xxhash library with the value + * of @ref XXH_NAMESPACE (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ + #define XXH_NAMESPACE /* YOUR NAME HERE */ + #undef XXH_NAMESPACE +#endif + #if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) && \ !defined(XXH_INLINE_ALL_31684351384) /* this section should be traversed only once */ @@ -160,6 +397,7 @@ extern "C" { #undef XXH3_64bits #undef XXH3_64bits_withSecret #undef XXH3_64bits_withSeed + #undef XXH3_64bits_withSecretandSeed #undef XXH3_createState #undef XXH3_freeState #undef XXH3_copyState @@ -177,6 +415,7 @@ extern "C" { #undef XXH3_128bits_reset #undef XXH3_128bits_reset_withSeed #undef XXH3_128bits_reset_withSecret + #undef XXH3_128bits_reset_withSecretandSeed #undef XXH3_128bits_update #undef XXH3_128bits_digest #undef XXH128_isEqual @@ -220,13 +459,7 @@ extern "C" { #ifndef XXHASH_H_5627135585666179 #define XXHASH_H_5627135585666179 1 - /*! - * @defgroup public Public API - * Contains details on the public xxHash functions. - * @{ - - */ - /* specific declaration modes for Windows */ + /*! @brief Marks a global symbol. */ #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) #if defined(WIN32) && defined(_MSC_VER) && \ (defined(XXH_IMPORT) || defined(XXH_EXPORT)) @@ -240,24 +473,6 @@ extern "C" { #endif #endif - #ifdef XXH_DOXYGEN - /*! - * @brief Emulate a namespace by transparently prefixing all symbols. - * - * If you want to include _and expose_ xxHash functions from within your own - * library, but also want to avoid symbol collisions with other libraries - * which may also include xxHash, you can use XXH_NAMESPACE to automatically - * prefix any public symbol from xxhash library with the value of - * XXH_NAMESPACE (therefore, avoid empty or numeric values). - * - * Note that no change is required within the calling program as long as it - * includes `xxhash.h`: Regular symbol names will be automatically - * translated by this header. - */ - #define XXH_NAMESPACE /* YOUR NAME HERE */ - #undef XXH_NAMESPACE - #endif - #ifdef XXH_NAMESPACE #define XXH_CAT(A, B) A##B #define XXH_NAME2(A, B) XXH_CAT(A, B) @@ -291,6 +506,8 @@ extern "C" { #define XXH3_64bits_withSecret \ XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) #define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) + #define XXH3_64bits_withSecretandSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) #define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) #define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) #define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) @@ -299,9 +516,13 @@ extern "C" { XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) #define XXH3_64bits_reset_withSecret \ XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) + #define XXH3_64bits_reset_withSecretandSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) #define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) #define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) #define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) + #define XXH3_generateSecret_fromSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) /* XXH3_128bits */ #define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) #define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) @@ -309,11 +530,15 @@ extern "C" { XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) #define XXH3_128bits_withSecret \ XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) + #define XXH3_128bits_withSecretandSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) #define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) #define XXH3_128bits_reset_withSeed \ XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) #define XXH3_128bits_reset_withSecret \ XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) + #define XXH3_128bits_reset_withSecretandSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) #define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) #define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) #define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) @@ -325,11 +550,40 @@ extern "C" { #endif /* ************************************* + * Compiler specifics + ***************************************/ + + /* specific declaration modes for Windows */ + #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) + #if defined(WIN32) && defined(_MSC_VER) && \ + (defined(XXH_IMPORT) || defined(XXH_EXPORT)) + #ifdef XXH_EXPORT + #define XXH_PUBLIC_API __declspec(dllexport) + #elif XXH_IMPORT + #define XXH_PUBLIC_API __declspec(dllimport) + #endif + #else + #define XXH_PUBLIC_API /* do nothing */ + #endif + #endif + + #if defined(__GNUC__) + #define XXH_CONSTF __attribute__((const)) + #define XXH_PUREF __attribute__((pure)) + #define XXH_MALLOCF __attribute__((malloc)) + #else + #define XXH_CONSTF /* disable */ + #define XXH_PUREF + #define XXH_MALLOCF + #endif + + /* ************************************* * Version ***************************************/ #define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MINOR 8 - #define XXH_VERSION_RELEASE 1 + #define XXH_VERSION_RELEASE 2 + /*! @brief Version number, encoded as two digits each */ #define XXH_VERSION_NUMBER \ (XXH_VERSION_MAJOR * 100 * 100 + XXH_VERSION_MINOR * 100 + \ XXH_VERSION_RELEASE) @@ -337,18 +591,26 @@ extern "C" { /*! * @brief Obtains the xxHash version. * - * This is only useful when xxHash is compiled as a shared library, as it is - * independent of the version defined in the header. + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. * - * @return `XXH_VERSION_NUMBER` as of when the libray was compiled. + * @return @ref XXH_VERSION_NUMBER of the invoked library. */ -XXH_PUBLIC_API unsigned XXH_versionNumber(void); +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber(void); /* **************************** - * Definitions + * Common basic types ******************************/ #include <stddef.h> /* size_t */ -typedef enum { XXH_OK = 0, XXH_ERROR } XXH_errorcode; +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ + +} XXH_errorcode; /*-********************************************************************** * 32-bit hash @@ -365,36 +627,33 @@ typedef uint32_t XXH32_hash_t; (defined(__cplusplus) || \ (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) #include <stdint.h> -typedef uint32_t XXH32_hash_t; +typedef uint32_t XXH32_hash_t; #else #include <limits.h> #if UINT_MAX == 0xFFFFFFFFUL typedef unsigned int XXH32_hash_t; - #else - #if ULONG_MAX == 0xFFFFFFFFUL + #elif ULONG_MAX == 0xFFFFFFFFUL typedef unsigned long XXH32_hash_t; - #else - #error "unsupported platform: need a 32-bit type" - #endif + #else + #error "unsupported platform: need a 32-bit type" #endif #endif /*! * @} * - * @defgroup xxh32_family XXH32 family + * @defgroup XXH32_family XXH32 family * @ingroup public * Contains functions used in the classic 32-bit xxHash algorithm. * * @note - * XXH32 is considered rather weak by today's standards. - * The @ref xxh3_family provides competitive speed for both 32-bit and 64-bit - * systems, and offers true 64/128 bit hash results. It provides a superior - * level of dispersion, and greatly reduces the risks of collisions. + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. * - * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families - * @see @ref xxh32_impl for implementation details + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details * @{ */ @@ -402,8 +661,6 @@ typedef unsigned long XXH32_hash_t; /*! * @brief Calculates the 32-bit hash of @p input using xxHash32. * - * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s - * * @param input The block of data to be hashed, at least @p length bytes in * size. * @param length The length of @p input, in bytes. @@ -414,92 +671,46 @@ typedef unsigned long XXH32_hash_t; * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * @return The calculated 32-bit hash value. - * - * @see - * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): - * Direct equivalents for the other variants of xxHash. - * @see - * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. - */ -XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t length, - XXH32_hash_t seed); - -/*! - * Streaming functions generate the xxHash value from an incremental input. - * This method is slower than single-call functions, due to state management. - * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. - * - * An XXH state must first be allocated using `XXH*_createState()`. - * - * Start a new hash by initializing the state with a seed using `XXH*_reset()`. - * - * Then, feed the hash state by calling `XXH*_update()` as many times as - * necessary. - * - * The function returns an error code, with 0 meaning OK, and any other value - * meaning there is an error. - * - * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. - * This function returns the nn-bits hash as an int or long long. - * - * It's still possible to continue inserting input into the hash state after a - * digest, and generate new hash values later on by invoking `XXH*_digest()`. - * - * When done, release the state using `XXH*_freeState()`. - * - * Example code for incrementally hashing a file: - * @code{.c} - * #include <stdio.h> - * #include <xxhash.h> - * #define BUFFER_SIZE 256 - * - * // Note: XXH64 and XXH3 use the same interface. - * XXH32_hash_t - * hashFile(FILE* stream) - * { - - * XXH32_state_t* state; - * unsigned char buf[BUFFER_SIZE]; - * size_t amt; - * XXH32_hash_t hash; + * @return The calculated 32-bit xxHash32 value. * - * state = XXH32_createState(); // Create a state - * assert(state != NULL); // Error check here - * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed - * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { - - * XXH32_update(state, buf, amt); // Hash the file in chunks - * } - * hash = XXH32_digest(state); // Finalize the hash - * XXH32_freeState(state); // Clean up - * return hash; - * } - * @endcode + * @see @ref single_shot_example "Single Shot Example" for an example. */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32(const void *input, size_t length, + XXH32_hash_t seed); + #ifndef XXH_NO_STREAM /*! * @typedef struct XXH32_state_s XXH32_state_t * @brief The opaque state struct for the XXH32 streaming API. * * @see XXH32_state_s for details. + * @see @ref streaming_example "Streaming Example" */ typedef struct XXH32_state_s XXH32_state_t; /*! * @brief Allocates an @ref XXH32_state_t. * - * Must be freed with XXH32_freeState(). - * @return An allocated XXH32_state_t on success, `NULL` on failure. + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + * + * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH32_state_t *XXH32_createState(void); +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t *XXH32_createState(void); /*! * @brief Frees an @ref XXH32_state_t. * - * Must be allocated with XXH32_createState(). * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref * XXH32_createState(). - * @return XXH_OK. + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + * @see @ref streaming_example "Streaming Example" + * */ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr); /*! @@ -516,15 +727,19 @@ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t *dst_state, /*! * @brief Resets an @ref XXH32_state_t to begin a new hash. * - * This function resets and seeds a state. Call it before @ref XXH32_update(). - * * @param statePtr The state struct to reset. * @param seed The 32-bit seed to alter the hash result predictably. * * @pre * @p statePtr must not be `NULL`. * - * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref + * XXH32_update(). + * + * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, XXH32_hash_t seed); @@ -532,8 +747,6 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, /*! * @brief Consumes a block of @p input to an @ref XXH32_state_t. * - * Call this to incrementally consume blocks of data. - * * @param statePtr The state struct to update. * @param input The block of data to be hashed, at least @p length bytes in * size. @@ -546,7 +759,12 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *statePtr, const void *input, size_t length); @@ -554,41 +772,24 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *statePtr, /*! * @brief Returns the calculated hash value from an @ref XXH32_state_t. * - * @note - * Calling XXH32_digest() will not affect @p statePtr, so you can update, - * digest, and update again. - * * @param statePtr The state struct to calculate the hash from. * * @pre * @p statePtr must not be `NULL`. * - * @return The calculated xxHash32 value from that state. - */ -XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *statePtr); - -/******* Canonical representation *******/ - -/* - * The default return values from XXH functions are unsigned 32 and 64 bit - * integers. - * This the simplest and fastest format for further post-processing. + * @return The calculated 32-bit xxHash32 value from that state. * - * However, this leaves open the question of what is the order on the byte - * level, since little and big endian conventions will store the same number - * differently. - * - * The canonical representation settles this issue by mandating big-endian - * convention, the same convention as human-readable numbers (large digits - * first). - * - * When writing hash values to storage, sending them over a network, or printing - * them, it's highly recommended to use the canonical representation to ensure - * portability across a wider range of systems, present and future. + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. * - * The following functions allow transformation of hash values to and from - * canonical format. + * @see @ref streaming_example "Streaming Example" */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t +XXH32_digest(const XXH32_state_t *statePtr); + #endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ /*! * @brief Canonical (big endian) representation of @ref XXH32_hash_t. @@ -602,11 +803,13 @@ typedef struct { /*! * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. * - * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param dst The @ref XXH32_canonical_t pointer to be stored to. * @param hash The @ref XXH32_hash_t to be converted. * * @pre * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, XXH32_hash_t hash); @@ -620,45 +823,75 @@ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, * @p src must not be `NULL`. * * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" */ -XXH_PUBLIC_API XXH32_hash_t +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t *src); + /*! @cond Doxygen ignores this part */ #ifdef __has_attribute #define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) #else #define XXH_HAS_ATTRIBUTE(x) 0 #endif + /*! @endcond */ + + /*! @cond Doxygen ignores this part */ + /* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ + #define XXH_C23_VN 201711L + /*! @endcond */ + /*! @cond Doxygen ignores this part */ /* C-language Attributes are added in C23. */ - #if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && \ + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && \ defined(__has_c_attribute) #define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) #else #define XXH_HAS_C_ATTRIBUTE(x) 0 #endif + /*! @endcond */ + /*! @cond Doxygen ignores this part */ #if defined(__cplusplus) && defined(__has_cpp_attribute) #define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) #else #define XXH_HAS_CPP_ATTRIBUTE(x) 0 #endif + /*! @endcond */ + /*! @cond Doxygen ignores this part */ /* - Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' - attribute introduced in CPP17 and C23. CPP17 : - https://en.cppreference.com/w/cpp/language/attributes/fallthrough C23 : - https://en.cppreference.com/w/c/language/attributes/fallthrough - */ - #if XXH_HAS_C_ATTRIBUTE(x) - #define XXH_FALLTHROUGH [[fallthrough]] - #elif XXH_HAS_CPP_ATTRIBUTE(x) + * Define XXH_FALLTHROUGH macro for annotating switch case with the + * 'fallthrough' attribute introduced in CPP17 and C23. CPP17 : + * https://en.cppreference.com/w/cpp/language/attributes/fallthrough C23 : + * https://en.cppreference.com/w/c/language/attributes/fallthrough + */ + #if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) #define XXH_FALLTHROUGH [[fallthrough]] #elif XXH_HAS_ATTRIBUTE(__fallthrough__) - #define XXH_FALLTHROUGH __attribute__((fallthrough)) + #define XXH_FALLTHROUGH __attribute__((__fallthrough__)) #else - #define XXH_FALLTHROUGH + #define XXH_FALLTHROUGH /* fallthrough */ #endif + /*! @endcond */ + + /*! @cond Doxygen ignores this part */ + /* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ + #if XXH_HAS_ATTRIBUTE(noescape) + #define XXH_NOESCAPE __attribute__((noescape)) + #else + #define XXH_NOESCAPE + #endif +/*! @endcond */ /*! * @} @@ -697,7 +930,7 @@ typedef unsigned long long XXH64_hash_t; /*! * @} * - * @defgroup xxh64_family XXH64 family + * @defgroup XXH64_family XXH64 family * @ingroup public * @{ @@ -705,16 +938,13 @@ typedef unsigned long long XXH64_hash_t; * * @note * XXH3 provides competitive speed for both 32-bit and 64-bit systems, - * and offers true 64/128 bit hash results. It provides a superior level of - * dispersion, and greatly reduces the risks of collisions. + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. */ /*! * @brief Calculates the 64-bit hash of @p input using xxHash64. * - * This function usually runs faster on 64-bit systems, but slower on 32-bit - * systems (see benchmark). - * * @param input The block of data to be hashed, at least @p length bytes in * size. * @param length The length of @p input, in bytes. @@ -725,51 +955,171 @@ typedef unsigned long long XXH64_hash_t; * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * @return The calculated 64-bit hash. + * @return The calculated 64-bit xxHash64 value. * - * @see - * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): - * Direct equivalents for the other variants of xxHash. - * @see - * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t length, - XXH64_hash_t seed); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void *input, + size_t length, XXH64_hash_t seed); -/******* Streaming *******/ + /******* Streaming *******/ + #ifndef XXH_NO_STREAM /*! * @brief The opaque state struct for the XXH64 streaming API. * * @see XXH64_state_s for details. + * @see @ref streaming_example "Streaming Example" */ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH64_state_t *XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t *dst_state, - const XXH64_state_t *src_state); +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t *statePtr, - XXH64_hash_t seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *statePtr, - const void *input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t *statePtr); +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH64_freeState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t *XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref + * XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t *dst_state, + const XXH64_state_t *src_state); +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref + * XXH64_update(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t *statePtr, + XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH_NOESCAPE XXH64_state_t *statePtr, + XXH_NOESCAPE const void *input, + size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH64_digest(XXH_NOESCAPE const XXH64_state_t *statePtr); + #endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, - XXH64_hash_t hash); -XXH_PUBLIC_API XXH64_hash_t -XXH64_hashFromCanonical(const XXH64_canonical_t *src); +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t *dst, + XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t *src); + + #ifndef XXH_NO_XXH3 /*! * @} * ************************************************************************ - * @defgroup xxh3_family XXH3 family + * @defgroup XXH3_family XXH3 family * @ingroup public * @{ @@ -790,16 +1140,27 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src); * * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, * but does not require it. - * Any 32-bit and 64-bit targets that can run XXH32 smoothly - * can run XXH3 at competitive speeds, even without vector support. - * Further details are explained in the implementation. - * - * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, - * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, + * an automatic runtime dispatcher is included separately in @ref + * xxh_x86dispatch.c. * * XXH3 implementation is portable: * it has a generic C90 formulation that can be compiled on any platform, - * all implementations generage exactly the same hash value on all platforms. + * all implementations generate exactly the same hash value on all platforms. * Starting from v0.8.0, it's also labelled "stable", meaning that * any future version will also generate the same hash value. * @@ -811,53 +1172,112 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src); * * The API supports one-shot hashing, streaming mode, and custom secrets. */ - /*-********************************************************************** * XXH3 64-bit variant ************************************************************************/ -/* XXH3_64bits(): - * default 64-bit variant, using default secret and default seed of 0. - * It's the fastest variant. */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *data, size_t len); +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, + * however it may have slightly better performance due to constant propagation + * of the defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits(XXH_NOESCAPE const void *input, size_t length); -/* - * XXH3_64bits_withSeed(): - * This variant generates a custom secret on the fly - * based on default secret altered using the `seed` value. +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * * While this operation is decently fast, note that it's not completely free. - * Note: seed==0 produces the same results as XXH3_64bits(). + * + * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void *data, size_t len, - XXH64_hash_t seed); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed( + XXH_NOESCAPE const void *input, size_t length, XXH64_hash_t seed); - /*! - * The bare minimum size for a custom secret. - * - * @see - * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), - * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). - */ - #define XXH3_SECRET_SIZE_MIN 136 + /*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ + #define XXH3_SECRET_SIZE_MIN 136 -/* - * XXH3_64bits_withSecret(): +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in + * size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * * It's possible to provide any blob of bytes as a "secret" to generate the * hash. This makes it more difficult for an external actor to prepare an - * intentional collision. The main condition is that secretSize *must* be large - * enough (>= XXH3_SECRET_SIZE_MIN). However, the quality of produced hash - * values depends on secret's entropy. Technically, the secret must look like a - * bunch of random bytes. Avoid "trivial" or structured data such as repeated - * sequences or a text document. Whenever unsure about the "randomness" of the - * blob of bytes, consider relabelling it as a "custom seed" instead, and employ - * "XXH3_generateSecret()" (see below) to generate a high entropy secret derived - * from the custom seed. - */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *data, size_t len, - const void *secret, - size_t secretSize); - -/******* Streaming *******/ + * intentional collision. The main condition is that @p secretSize *must* be + * large enough (>= @ref XXH3_SECRET_SIZE_MIN). However, the quality of the + * secret impacts the dispersion of the hash algorithm. Therefore, the secret + * _must_ look like a bunch of random bytes. Avoid "trivial" or structured data + * such as repeated sequences or a text document. Whenever in doubt about the + * "randomness" of the blob of bytes, consider employing @ref + * XXH3_generateSecret() instead (see below). It will generate a proper high + * entropy secret derived from the blob of bytes. Another advantage of using + * XXH3_generateSecret() is that it guarantees that all bits within the initial + * blob of bytes will impact every bit of the output. This is not necessarily + * the case when using the blob of bytes directly because, when hashing _small_ + * inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void *data, size_t len, + XXH_NOESCAPE const void *secret, size_t secretSize); + + /******* Streaming *******/ + #ifndef XXH_NO_STREAM /* * Streaming requires state maintenance. * This operation costs memory and CPU. @@ -866,45 +1286,146 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *data, size_t len, */ /*! - * @brief The state struct for the XXH3 streaming API. + * @brief The opaque state struct for the XXH3 streaming API. * * @see XXH3_state_s for details. + * @see @ref streaming_example "Streaming Example" */ -typedef struct XXH3_state_s XXH3_state_t; -XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr); -XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t *dst_state, - const XXH3_state_t *src_state); +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t *XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr); -/* - * XXH3_64bits_reset(): - * Initialize with default parameters. - * digest will be equivalent to `XXH3_64bits()`. +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t *statePtr); -/* - * XXH3_64bits_reset_withSeed(): - * Generate a custom secret from `seed`, and store it into `statePtr`. - * digest will be equivalent to `XXH3_64bits_withSeed()`. +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t *dst_state, + XXH_NOESCAPE const XXH3_state_t *src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default + * parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + * @see @ref streaming_example "Streaming Example" + * */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t *statePtr, - XXH64_hash_t seed); -/* - * XXH3_64bits_reset_withSecret(): - * `secret` is referenced, it _must outlive_ the hash streaming session. - * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t *statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + * @see @ref streaming_example "Streaming Example" + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH64_hash_t seed); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, * and the quality of produced hash values depends on secret's entropy * (secret's content should look like a bunch of random bytes). * When in doubt about the randomness of a candidate `secret`, * consider employing `XXH3_generateSecret()` instead (see below). + * + * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret( - XXH3_state_t *statePtr, const void *secret, size_t secretSize); + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t *statePtr, + XXH_NOESCAPE const void *input, size_t length); -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update(XXH3_state_t *statePtr, - const void *input, - size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *statePtr); +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref + * XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can + * update, digest, and update again. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_digest(XXH_NOESCAPE const XXH3_state_t *statePtr); + #endif /* !XXH_NO_STREAM */ /* note : canonical representation of XXH3 is the same as XXH64 * since they both produce XXH64_hash_t values */ @@ -926,15 +1447,82 @@ typedef struct { } XXH128_hash_t; -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void *data, size_t len); -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void *data, size_t len, - XXH64_hash_t seed); -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *data, - size_t len, - const void *secret, - size_t secretSize); +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, + * however it may have slightly better performance due to constant propagation + * of the defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding + * variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits(XXH_NOESCAPE const void *data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed( + XXH_NOESCAPE const void *data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in + * size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the + * hash. This makes it more difficult for an external actor to prepare an + * intentional collision. The main condition is that @p secretSize *must* be + * large enough (>= @ref XXH3_SECRET_SIZE_MIN). However, the quality of the + * secret impacts the dispersion of the hash algorithm. Therefore, the secret + * _must_ look like a bunch of random bytes. Avoid "trivial" or structured data + * such as repeated sequences or a text document. Whenever in doubt about the + * "randomness" of the blob of bytes, consider employing @ref + * XXH3_generateSecret() instead (see below). It will generate a proper high + * entropy secret derived from the blob of bytes. Another advantage of using + * XXH3_generateSecret() is that it guarantees that all bits within the initial + * blob of bytes will impact every bit of the output. This is not necessarily + * the case when using the blob of bytes directly because, when hashing _small_ + * inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void *data, size_t len, + XXH_NOESCAPE const void *secret, size_t secretSize); -/******* Streaming *******/ + /******* Streaming *******/ + #ifndef XXH_NO_STREAM /* * Streaming requires state maintenance. * This operation costs memory and CPU. @@ -948,39 +1536,151 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *data, * counterpart. */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t *statePtr); -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t *statePtr, - XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default + * parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t *statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + * + * @see @ref streaming_example "Streaming Example" + */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret( - XXH3_state_t *statePtr, const void *secret, size_t secretSize); + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t *statePtr, + XXH_NOESCAPE const void *input, size_t length); -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t *statePtr, - const void *input, - size_t length); -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t *statePtr); +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref + * XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can + * update, digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_digest(XXH_NOESCAPE const XXH3_state_t *statePtr); + #endif /* !XXH_NO_STREAM */ /* Following helper functions make it possible to compare XXH128_hast_t values. * Since XXH128_hash_t is a structure, this capability is not offered by the - * language. - * Note: For better performance, these functions can be inlined using + * language. Note: For better performance, these functions can be inlined using * XXH_INLINE_ALL */ /*! - * XXH128_isEqual(): - * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. */ -XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); /*! - * XXH128_cmp(): + * @brief Compares two @ref XXH128_hash_t * * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. * - * return: >0 if *h128_1 > *h128_2 - * =0 if *h128_1 == *h128_2 - * <0 if *h128_1 < *h128_2 + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 */ -XXH_PUBLIC_API int XXH128_cmp(const void *h128_1, const void *h128_2); +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void *h128_1, + XXH_NOESCAPE const void *h128_2); /******* Canonical representation *******/ typedef struct { @@ -989,11 +1689,35 @@ typedef struct { } XXH128_canonical_t; -XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t *dst, - XXH128_hash_t hash); -XXH_PUBLIC_API XXH128_hash_t -XXH128_hashFromCanonical(const XXH128_canonical_t *src); +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref + * XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash( + XXH_NOESCAPE XXH128_canonical_t *dst, XXH128_hash_t hash); +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t *src); + + #endif /* !XXH_NO_XXH3 */ #endif /* XXH_NO_LONG_LONG */ /*! @@ -1035,15 +1759,11 @@ struct XXH32_state_s { XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ - XXH32_hash_t v1; /*!< First accumulator lane */ - XXH32_hash_t v2; /*!< Second accumulator lane */ - XXH32_hash_t v3; /*!< Third accumulator lane */ - XXH32_hash_t v4; /*!< Fourth accumulator lane */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ - XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may - be removed. */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ }; /* typedef'd to XXH32_state_t */ @@ -1064,62 +1784,62 @@ struct XXH32_state_s { struct XXH64_state_s { XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ - XXH64_hash_t v1; /*!< First accumulator lane */ - XXH64_hash_t v2; /*!< Second accumulator lane */ - XXH64_hash_t v3; /*!< Third accumulator lane */ - XXH64_hash_t v4; /*!< Fourth accumulator lane */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ - XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it - may be removed. */ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ }; /* typedef'd to XXH64_state_t */ - #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 \ - */ - #include <stdalign.h> - #define XXH_ALIGN(n) alignas(n) - #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ + #ifndef XXH_NO_XXH3 + + #if defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L) /* >= C11 */ + #include <stdalign.h> + #define XXH_ALIGN(n) alignas(n) + #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ /* In C++ alignas() is a keyword */ - #define XXH_ALIGN(n) alignas(n) - #elif defined(__GNUC__) - #define XXH_ALIGN(n) __attribute__((aligned(n))) - #elif defined(_MSC_VER) - #define XXH_ALIGN(n) __declspec(align(n)) - #else - #define XXH_ALIGN(n) /* disabled */ - #endif + #define XXH_ALIGN(n) alignas(n) + #elif defined(__GNUC__) + #define XXH_ALIGN(n) __attribute__((aligned(n))) + #elif defined(_MSC_VER) + #define XXH_ALIGN(n) __declspec(align(n)) + #else + #define XXH_ALIGN(n) /* disabled */ + #endif - /* Old GCC versions only accept the attribute after the type in structures. - */ - #if !(defined(__STDC_VERSION__) && \ - (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ - && !(defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ - && defined(__GNUC__) - #define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) - #else - #define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type - #endif + /* Old GCC versions only accept the attribute after the type in + * structures. */ + #if !(defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && \ + !(defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) + #define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) + #else + #define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type + #endif - /*! - * @brief The size of the internal XXH3 buffer. - * - * This is the optimal update size for incremental hashing. - * - * @see XXH3_64b_update(), XXH3_128b_update(). - */ - #define XXH3_INTERNALBUFFER_SIZE 256 + /*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ + #define XXH3_INTERNALBUFFER_SIZE 256 - /*! - * @brief Default size of the secret buffer (and @ref XXH3_kSecret). - * - * This is the size used in @ref XXH3_kSecret and the seeded functions. - * - * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. - */ - #define XXH3_SECRET_DEFAULT_SIZE 192 + /*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ + #define XXH3_SECRET_DEFAULT_SIZE 192 /*! * @internal @@ -1146,15 +1866,15 @@ struct XXH64_state_s { struct XXH3_state_s { XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); - /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref - * XXH64_state_s */ + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v + */ XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); /*!< Used to store a custom secret generated from a seed. */ XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); /*!< The internal buffer. @see XXH32_state_s::mem32 */ XXH32_hash_t bufferedSize; /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ - XXH32_hash_t reserved32; + XXH32_hash_t useSeed; /*!< Reserved field. Needed for padding on 64-bit. */ size_t nbStripesSoFar; /*!< Number or stripes processed. */ @@ -1176,67 +1896,262 @@ struct XXH3_state_s { }; /* typedef'd to XXH3_state_t */ - #undef XXH_ALIGN_MEMBER + #undef XXH_ALIGN_MEMBER - /*! - * @brief Initializes a stack-allocated `XXH3_state_s`. - * - * When the @ref XXH3_state_t structure is merely emplaced on stack, - * it should be initialized with XXH3_INITSTATE() or a memset() - * in case its first reset uses XXH3_NNbits_reset_withSeed(). - * This init can be omitted if the first reset uses default or _withSecret - * mode. This operation isn't necessary when the state is created with - * XXH3_createState(). Note that this doesn't prepare the state for a - * streaming operation, it's still necessary to use XXH3_NNbits_reset*() - * afterwards. - */ - #define XXH3_INITSTATE(XXH3_state_ptr) \ - { (XXH3_state_ptr)->seed = 0; } + /*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret + * mode. This operation isn't necessary when the state is created with + * XXH3_createState(). Note that this doesn't prepare the state for a + * streaming operation, it's still necessary to use XXH3_NNbits_reset*() + * afterwards. + */ + #define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + \ + XXH3_state_t *tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + \ + } while (0) + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void *data, + size_t len, XXH64_hash_t seed); /* === Experimental API === */ /* Symbols defined below must be considered tied to a specific library version. */ -/* - * XXH3_generateSecret(): +/*! + * @brief Derive a high-entropy secret from any user-defined content, named + * customSeed. + * + * @param secretBuffer A writable buffer for derived high-entropy secret + * data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= + * XXH3_SECRET_DEFAULT_SIZE. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. * - * Derive a high-entropy secret from any user-defined content, named customSeed. * The generated secret can be used in combination with `*_withSecret()` * functions. The `_withSecret()` variants are useful to provide a higher level * of protection than 64-bit seed, as it becomes much more difficult for an * external actor to guess how to impact the calculation logic. * * The function accepts as input a custom seed of any length and any content, - * and derives from it a high-entropy secret of length XXH3_SECRET_DEFAULT_SIZE - * into an already allocated buffer secretBuffer. - * The generated secret is _always_ XXH_SECRET_DEFAULT_SIZE bytes long. + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. * * The generated secret can then be used with any `*_withSecret()` variant. - * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, - * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() * are part of this list. They all accept a `secret` parameter - * which must be very long for implementation reasons (>= XXH3_SECRET_SIZE_MIN) - * _and_ feature very high entropy (consist of random-looking bytes). - * These conditions can be a high bar to meet, so - * this function can be used to generate a secret of proper quality. - * - * customSeed can be anything. It can have any size, even small ones, - * and its content can be anything, even stupidly "low entropy" source such as a - * bunch of zeroes. The resulting `secret` will nonetheless provide all expected + * which must be large enough for implementation reasons (>= @ref + * XXH3_SECRET_SIZE_MIN) _and_ feature very high entropy (consist of + * random-looking bytes). These conditions can be a high bar to meet, so @ref + * XXH3_generateSecret() can be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required * qualities. * - * Supplying NULL as the customSeed copies the default secret into - * `secretBuffer`. When customSeedSize > 0, supplying NULL as customSeed is - * undefined behavior. + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined + * behavior. + * + * Example code: + * @code{.c} + * #include <stdio.h> + * #include <stdlib.h> + * #include <string.h> + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode */ -XXH_PUBLIC_API void XXH3_generateSecret(void *secretBuffer, - const void *customSeed, - size_t customSeedSize); +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void *secretBuffer, size_t secretSize, + XXH_NOESCAPE const void *customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include <string> + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { -/* simple short-cut to pre-selected XXH3_128bits variant */ -XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, - XXH64_hash_t seed); + * return size_t{ + + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, + *sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed( + XXH_NOESCAPE void *secretBuffer, XXH64_hash_t seed); + +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in + * size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either + * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" + * scenarios, because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecretandSeed( + XXH_NOESCAPE const void *data, size_t len, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param input The block of data to be hashed, at least @p len bytes in + * size. + * @param length The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecretandSeed( + XXH_NOESCAPE const void *input, size_t length, + XXH_NOESCAPE const void *secret, size_t secretSize, XXH64_hash_t seed64); + #ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref + * XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecretandSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed64); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref + * XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecretandSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed64); + #endif /* !XXH_NO_STREAM */ + #endif /* !XXH_NO_XXH3 */ #endif /* XXH_NO_LONG_LONG */ #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) #define XXH_IMPLEMENTATION @@ -1291,7 +2206,7 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, /*! * @brief Define this to disable 64-bit code. * - * Useful if only using the @ref xxh32_family and you have a strict C90 + * Useful if only using the @ref XXH32_family and you have a strict C90 * compiler. */ #define XXH_NO_LONG_LONG @@ -1315,7 +2230,7 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, * Use `memcpy()`. Safe and portable. Note that most modern compilers * will eliminate the function call and treat it as an unaligned access. * - * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` * @par * Depends on compiler extensions and is therefore not portable. * This method is safe _if_ your compiler supports it, @@ -1336,30 +2251,50 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, * big-endian systems which lack a native byteswap instruction. However, * some compilers will emit literal byteshifts even if the target supports * unaligned access. - * . + * * * @warning * Methods 1 and 2 rely on implementation-defined behavior. Use these with * care, as what works on one compiler/platform/optimization level may * cause another to read garbage data or even crash. * - * See https://stackoverflow.com/a/32095106/646947 for details. + * See + * https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + * for details. * * Prefer these methods in priority order (0 > 3 > 1 > 2) */ #define XXH_FORCE_MEMORY_ACCESS 0 + /*! - * @def XXH_ACCEPT_NULL_INPUT_POINTER - * @brief Whether to add explicit `NULL` checks. + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. * - * If the input pointer is `NULL` and the length is non-zero, xxHash's - * default behavior is to dereference it, triggering a segfault. + * xxHash, when compiled, tends to result in a rather large binary size. + * This is mostly due to heavy usage to forced inlining and constant folding + * of the + * @ref XXH3_family to increase performance. * - * When this macro is enabled, xxHash actively checks the input for a null - * pointer. If it is, the result for null input pointers is the same as a - * zero-length input. + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. + * Speed comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies + * the options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == + * 0, and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use + * the streaming API. */ - #define XXH_ACCEPT_NULL_INPUT_POINTER 0 + #define XXH_SIZE_OPT 0 + /*! * @def XXH_FORCE_ALIGN_CHECK * @brief If defined to non-zero, adds a special path for aligned inputs @@ -1381,8 +2316,10 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, * * In these cases, the alignment check can be removed by setting this macro * to 0. Then the code will always use unaligned memory access. Align check - * is automatically disabled on x86, x64 & arm64, which are platforms known - * to offer good unaligned memory accesses performance. + * is automatically disabled on x86, x64, ARM64, and some ARM chips which + * are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. * * This option does not affect XXH3 (only XXH32 and XXH64). */ @@ -1405,24 +2342,39 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the * compiler full control on whether to inline or not. * - * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using - * -fno-inline with GCC or Clang, this will automatically be defined. + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. */ #define XXH_NO_INLINE_HINTS 0 /*! - * @def XXH_REROLL - * @brief Whether to reroll `XXH32_finalize`. + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). * - * For performance, `XXH32_finalize` uses an unrolled loop - * in the form of a switch statement. + * However, if the secret size is not known, it doesn't have any benefit. + * This happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. * - * This is not always desirable, as it generates larger code, - * and depending on the architecture, may even be slower + * Additionally, this defaults to 0 on GCC 12+, which has an issue with + * function pointers that are *sometimes* force inline on -Og, and it is + * impossible to automatically detect this optimization level. + */ + #define XXH3_INLINE_SECRET 0 + + /*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. * - * This is automatically defined with `-Os`/`-Oz` on GCC and Clang. + * For performance, `XXH32_finalize` uses multiple branches in the + * finalizer. This is generally preferable for performance, but depending on + * exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. */ - #define XXH_REROLL 0 + #define XXH32_ENDJMP 0 /*! * @internal @@ -1434,6 +2386,18 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, */ #define XXH_OLD_NAMES #undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + + /*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, + * disabling the streaming functions can improve code size significantly, + * especially with the @ref XXH3_family which tends to make constant folded + * copies of itself. + */ + #define XXH_NO_STREAM + #undef XXH_NO_STREAM /* don't actually */ #endif /* XXH_DOXYGEN */ /*! * @} @@ -1441,24 +2405,31 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command \ line for example */ - /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ - #if !defined(__clang__) && \ - ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ - (defined(__GNUC__) && \ - ((defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ - (defined(__mips__) && (__mips <= 5 || __mips_isa_rev < 6) && \ - (!defined(__mips16) || defined(__mips_mips16e2)))))) + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte + * shifting, so we use memcpy which for some reason does unaligned loads. */ + #if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && \ + defined(__ARM_FEATURE_UNALIGNED)) #define XXH_FORCE_MEMORY_ACCESS 1 #endif #endif - #ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ - #define XXH_ACCEPT_NULL_INPUT_POINTER 0 + #ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ + #if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) + #define XXH_SIZE_OPT 1 + #else + #define XXH_SIZE_OPT 0 + #endif #endif #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ - #if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || \ - defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is + * available */ + #if XXH_SIZE_OPT >= 1 || defined(__i386) || defined(__x86_64__) || \ + defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) || \ + defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || \ + defined(_M_ARM) /* visual */ #define XXH_FORCE_ALIGN_CHECK 0 #else #define XXH_FORCE_ALIGN_CHECK 1 @@ -1466,24 +2437,27 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, #endif #ifndef XXH_NO_INLINE_HINTS - #if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ - || defined(__NO_INLINE__) /* -O0, -fno-inline */ + #if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ #define XXH_NO_INLINE_HINTS 1 #else #define XXH_NO_INLINE_HINTS 0 #endif #endif - #ifndef XXH_REROLL - #if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ || \ - (defined(__GNUC__) && !defined(__clang__)) - /* The if/then loop is preferable to switch/case on gcc (on x64) */ - #define XXH_REROLL 1 + #ifndef XXH3_INLINE_SECRET + #if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) || \ + !defined(XXH_INLINE_ALL) + #define XXH3_INLINE_SECRET 0 #else - #define XXH_REROLL 0 + #define XXH3_INLINE_SECRET 1 #endif #endif + #ifndef XXH32_ENDJMP + /* generally preferable for performance */ + #define XXH32_ENDJMP 0 + #endif + /*! * @defgroup impl Implementation * @{ @@ -1493,17 +2467,45 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, /* ************************************* * Includes & Memory related functions ***************************************/ + #if defined(XXH_NO_STREAM) + /* nothing */ + #elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +static XXH_CONSTF void *XXH_malloc(size_t s) { + + (void)s; + return NULL; + +} + +static void XXH_free(void *p) { + + (void)p; + +} + + #else + /* * Modify the local functions below should you wish to use * different memory routines for malloc() and free() */ - #include <stdlib.h> + #include <stdlib.h> /*! * @internal * @brief Modify this function to use a different routine than malloc(). */ -static void *XXH_malloc(size_t s) { +static XXH_MALLOCF void *XXH_malloc(size_t s) { return malloc(s); @@ -1519,6 +2521,8 @@ static void XXH_free(void *p) { } + #endif /* XXH_NO_STDLIB */ + #include <string.h> /*! @@ -1542,20 +2546,20 @@ static void *XXH_memcpy(void *dest, const void *src, size_t size) { #endif #if XXH_NO_INLINE_HINTS /* disable inlining hints */ - #if defined(__GNUC__) + #if defined(__GNUC__) || defined(__clang__) #define XXH_FORCE_INLINE static __attribute__((unused)) #else #define XXH_FORCE_INLINE static #endif #define XXH_NO_INLINE static /* enable inlining hints */ - #elif defined(_MSC_VER) /* Visual Studio */ - #define XXH_FORCE_INLINE static __forceinline - #define XXH_NO_INLINE static __declspec(noinline) - #elif defined(__GNUC__) + #elif defined(__GNUC__) || defined(__clang__) #define XXH_FORCE_INLINE \ static __inline__ __attribute__((always_inline, unused)) #define XXH_NO_INLINE static __attribute__((noinline)) + #elif defined(_MSC_VER) /* Visual Studio */ + #define XXH_FORCE_INLINE static __forceinline + #define XXH_NO_INLINE static __declspec(noinline) #elif defined(__cplusplus) || \ (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ #define XXH_FORCE_INLINE static inline @@ -1565,6 +2569,12 @@ static void *XXH_memcpy(void *dest, const void *src, size_t size) { #define XXH_NO_INLINE static #endif + #if XXH3_INLINE_SECRET + #define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE + #else + #define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE + #endif + /* ************************************* * Debug ***************************************/ @@ -1588,17 +2598,20 @@ static void *XXH_memcpy(void *dest, const void *src, size_t size) { #include <assert.h> /* note: can still be disabled with NDEBUG */ #define XXH_ASSERT(c) assert(c) #else - #define XXH_ASSERT(c) ((void)0) + #if defined(__INTEL_COMPILER) + #define XXH_ASSERT(c) XXH_ASSUME((unsigned char)(c)) + #else + #define XXH_ASSERT(c) XXH_ASSUME(c) + #endif #endif /* note: use after variable declarations */ #ifndef XXH_STATIC_ASSERT #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ - #include <assert.h> #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ do { \ \ - static_assert((c), m); \ + _Static_assert((c), m); \ \ } while (0) @@ -1642,12 +2655,20 @@ static void *XXH_memcpy(void *dest, const void *src, size_t size) { * We also use it to prevent unwanted constant folding for AArch64 in * XXH3_initCustomSecret_scalar(). */ - #ifdef __GNUC__ - #define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r"(var)) + #if defined(__GNUC__) || defined(__clang__) + #define XXH_COMPILER_GUARD(var) __asm__("" : "+r"(var)) #else #define XXH_COMPILER_GUARD(var) ((void)0) #endif + /* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ + #if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) + #define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w"(var)) + #else + #define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) + #endif + /* ************************************* * Basic Types ***************************************/ @@ -1662,6 +2683,8 @@ typedef unsigned char xxh_u8; typedef XXH32_hash_t xxh_u32; #ifdef XXH_OLD_NAMES + #warning \ + "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" #define BYTE xxh_u8 #define U8 xxh_u8 #define U32 xxh_u32 @@ -1739,10 +2762,11 @@ static xxh_u32 XXH_read32(const void *memPtr) { #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 1)) /* - * __pack instructions are safer but compiler specific, hence potentially - * problematic for some compilers. - * - * Currently only defined for GCC and ICC. + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually + * it can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. */ #ifdef XXH_OLD_NAMES typedef union { @@ -1754,13 +2778,8 @@ typedef union { #endif static xxh_u32 XXH_read32(const void *ptr) { - typedef union { - - xxh_u32 u32; - - } __attribute__((packed)) xxh_unalign; - - return ((const xxh_unalign *)ptr)->u32; + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32 *)ptr); } @@ -1768,12 +2787,13 @@ static xxh_u32 XXH_read32(const void *ptr) { /* * Portable and safe solution. Generally efficient. - * see: https://stackoverflow.com/a/32095106/646947 + * see: + * https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html */ static xxh_u32 XXH_read32(const void *memPtr) { xxh_u32 val; - memcpy(&val, memPtr, sizeof(val)); + XXH_memcpy(&val, memPtr, sizeof(val)); return val; } @@ -1849,6 +2869,50 @@ static int XXH_isLittleEndian(void) { #define XXH_HAS_BUILTIN(x) 0 #endif +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include <stddef.h> + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include <utility> + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `<version>` header. + * We don't use that as including `<utility>` in `extern "C"` blocks + * doesn't work on GCC12 + */ + + #if XXH_HAS_BUILTIN(__builtin_unreachable) + #define XXH_UNREACHABLE() __builtin_unreachable() + + #elif defined(_MSC_VER) + #define XXH_UNREACHABLE() __assume(0) + + #else + #define XXH_UNREACHABLE() + #endif + + #if XXH_HAS_BUILTIN(__builtin_assume) + #define XXH_ASSUME(c) __builtin_assume(c) + #else + #define XXH_ASSUME(c) \ + if (!(c)) { XXH_UNREACHABLE(); } + #endif + /*! * @internal * @def XXH_rotl32(x,r) @@ -1982,8 +3046,10 @@ XXH_PUBLIC_API unsigned XXH_versionNumber(void) { *********************************************************************/ /*! * @} - * @defgroup xxh32_impl XXH32 implementation + * @defgroup XXH32_impl XXH32 implementation * @ingroup impl + * + * Details on the XXH32 implementation. * @{ */ @@ -2018,7 +3084,8 @@ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { acc += input * XXH_PRIME32_2; acc = XXH_rotl32(acc, 13); acc *= XXH_PRIME32_1; - #if (defined(__SSE4_1__) || defined(__aarch64__)) && \ + #if (defined(__SSE4_1__) || defined(__aarch64__) || \ + defined(__wasm_simd128__)) && \ !defined(XXH_ENABLE_AUTOVECTORIZE) /* * UGLY HACK: @@ -2049,9 +3116,12 @@ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { * can load data, while v3 can multiply. SSE forces them to operate * together. * - * This is also enabled on AArch64, as Clang autovectorizes it incorrectly - * and it is pointless writing a NEON implementation that is basically the - * same speed as scalar for XXH32. + * This is also enabled on AArch64, as Clang is *very aggressive* in + * vectorizing the loop. NEON is only faster on the A53, and with the newer + * cores, it is less than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. */ XXH_COMPILER_GUARD(acc); #endif @@ -2066,39 +3136,22 @@ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { * The final mix ensures that all input bits have a chance to impact any bit in * the output digest, resulting in an unbiased distribution. * - * @param h32 The hash to avalanche. + * @param hash The hash to avalanche. * @return The avalanched hash. */ -static xxh_u32 XXH32_avalanche(xxh_u32 h32) { +static xxh_u32 XXH32_avalanche(xxh_u32 hash) { - h32 ^= h32 >> 15; - h32 *= XXH_PRIME32_2; - h32 ^= h32 >> 13; - h32 *= XXH_PRIME32_3; - h32 ^= h32 >> 16; - return (h32); + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; } #define XXH_get32bits(p) XXH_readLE32_align(p, align) - #define XXH_PROCESS1 \ - do { \ - \ - h32 += (*ptr++) * XXH_PRIME32_5; \ - h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ - \ - } while (0) - - #define XXH_PROCESS4 \ - do { \ - \ - h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ - ptr += 4; \ - h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ - \ - } while (0) - /*! * @internal * @brief Processes the last 0-15 bytes of @p ptr. @@ -2107,17 +3160,37 @@ static xxh_u32 XXH32_avalanche(xxh_u32 h32) { * This final stage will digest them to ensure that all input bytes are present * in the final mix. * - * @param h32 The hash to finalize. + * @param hash The hash to finalize. * @param ptr The pointer to the remaining input. * @param len The remaining length, modulo 16. * @param align Whether @p ptr is aligned. * @return The finalized hash. + * @see XXH64_finalize(). */ -static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, - XXH_alignment align) { +static XXH_PUREF xxh_u32 XXH32_finalize(xxh_u32 hash, const xxh_u8 *ptr, + size_t len, XXH_alignment align) { +\ + #define XXH_PROCESS1 \ + do { \ + \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ + \ + } while (0) + + #define XXH_PROCESS4 \ + do { \ + \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ + \ + } while (0) - /* Compact rerolled version */ - if (XXH_REROLL) { + if (ptr == NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { len &= 15; while (len >= 4) { @@ -2134,7 +3207,7 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, } - return XXH32_avalanche(h32); + return XXH32_avalanche(hash); } else { @@ -2142,62 +3215,62 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, case 12: XXH_PROCESS4; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 8: XXH_PROCESS4; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 4: XXH_PROCESS4; - return XXH32_avalanche(h32); + return XXH32_avalanche(hash); case 13: XXH_PROCESS4; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 9: XXH_PROCESS4; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 5: XXH_PROCESS4; XXH_PROCESS1; - return XXH32_avalanche(h32); + return XXH32_avalanche(hash); case 14: XXH_PROCESS4; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 10: XXH_PROCESS4; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 6: XXH_PROCESS4; XXH_PROCESS1; XXH_PROCESS1; - return XXH32_avalanche(h32); + return XXH32_avalanche(hash); case 15: XXH_PROCESS4; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 11: XXH_PROCESS4; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 7: XXH_PROCESS4; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 3: XXH_PROCESS1; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 2: XXH_PROCESS1; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 1: XXH_PROCESS1; - XXH_FALLTHROUGH; + XXH_FALLTHROUGH; /* fallthrough */ case 0: - return XXH32_avalanche(h32); + return XXH32_avalanche(hash); } XXH_ASSERT(0); - return h32; /* reaching this point is deemed impossible */ + return hash; /* reaching this point is deemed impossible */ } @@ -2215,29 +3288,21 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, * @internal * @brief The implementation for @ref XXH32(). * - * @param input, len, seed Directly passed from @ref XXH32(). + * @param input , len , seed Directly passed from @ref XXH32(). * @param align Whether @p input is aligned. * @return The calculated hash. */ -XXH_FORCE_INLINE xxh_u32 XXH32_endian_align(const xxh_u8 *input, size_t len, - xxh_u32 seed, XXH_alignment align) { - - const xxh_u8 *bEnd = input ? input + len : NULL; - xxh_u32 h32; - - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) - if (input == NULL) { - - len = 0; - bEnd = input = (const xxh_u8 *)(size_t)16; +XXH_FORCE_INLINE XXH_PUREF xxh_u32 XXH32_endian_align(const xxh_u8 *input, + size_t len, xxh_u32 seed, + XXH_alignment align) { - } + xxh_u32 h32; - #endif + if (input == NULL) XXH_ASSERT(len == 0); if (len >= 16) { + const xxh_u8 *const bEnd = input + len; const xxh_u8 *const limit = bEnd - 15; xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; xxh_u32 v2 = seed + XXH_PRIME32_2; @@ -2272,16 +3337,17 @@ XXH_FORCE_INLINE xxh_u32 XXH32_endian_align(const xxh_u8 *input, size_t len, } -/*! @ingroup xxh32_family */ +/*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, XXH32_hash_t seed) { - #if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, (const xxh_u8*)input, len); - return XXH32_digest(&state); + #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small + * inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8 *)input, len); + return XXH32_digest(&state); #else if (XXH_FORCE_ALIGN_CHECK) { @@ -2298,17 +3364,16 @@ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, } -/******* Hash streaming *******/ -/*! - * @ingroup xxh32_family - */ + /******* Hash streaming *******/ + #ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH32_state_t *XXH32_createState(void) { return (XXH32_state_t *)XXH_malloc(sizeof(XXH32_state_t)); } -/*! @ingroup xxh32_family */ +/*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr) { XXH_free(statePtr); @@ -2316,42 +3381,38 @@ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr) { } -/*! @ingroup xxh32_family */ +/*! @ingroup XXH32_family */ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t *dstState, const XXH32_state_t *srcState) { - memcpy(dstState, srcState, sizeof(*dstState)); + XXH_memcpy(dstState, srcState, sizeof(*dstState)); } -/*! @ingroup xxh32_family */ +/*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, XXH32_hash_t seed) { - XXH32_state_t state; /* using a local state to memcpy() in order to avoid - strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; - state.v2 = seed + XXH_PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - XXH_PRIME32_1; - /* do not write into reserved, planned to be removed in a future version */ - memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; return XXH_OK; } -/*! @ingroup xxh32_family */ +/*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, const void *input, size_t len) { - if (input == NULL) - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) + if (input == NULL) { + + XXH_ASSERT(len == 0); return XXH_OK; - #else - return XXH_ERROR; - #endif + + } { @@ -2375,13 +3436,13 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, { const xxh_u32 *p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32)); + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32)); + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32)); + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32)); + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); } @@ -2393,29 +3454,20 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, if (p <= bEnd - 16) { const xxh_u8 *const limit = bEnd - 16; - xxh_u32 v1 = state->v1; - xxh_u32 v2 = state->v2; - xxh_u32 v3 = state->v3; - xxh_u32 v4 = state->v4; do { - v1 = XXH32_round(v1, XXH_readLE32(p)); + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p += 4; - v2 = XXH32_round(v2, XXH_readLE32(p)); + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p += 4; - v3 = XXH32_round(v3, XXH_readLE32(p)); + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p += 4; - v4 = XXH32_round(v4, XXH_readLE32(p)); + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p += 4; } while (p <= limit); - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } if (p < bEnd) { @@ -2431,19 +3483,19 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, } -/*! @ingroup xxh32_family */ +/*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *state) { xxh_u32 h32; if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + - XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + h32 = XXH_rotl32(state->v[0], 1) + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + XXH_rotl32(state->v[3], 18); } else { - h32 = state->v3 /* == seed */ + XXH_PRIME32_5; + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; } @@ -2454,32 +3506,21 @@ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *state) { } + #endif /* !XXH_NO_STREAM */ + /******* Canonical representation *******/ -/*! - * @ingroup xxh32_family - * The default return values from XXH functions are unsigned 32 and 64 bit - * integers. - * - * The canonical representation uses big endian convention, the same convention - * as human-readable numbers (large digits first). - * - * This way, hash values can be written into a file or buffer, remaining - * comparable across different systems. - * - * The following functions allow transformation of hash values to and from their - * canonical format. - */ +/*! @ingroup XXH32_family */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, XXH32_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - memcpy(dst, &hash, sizeof(*dst)); + XXH_memcpy(dst, &hash, sizeof(*dst)); } -/*! @ingroup xxh32_family */ +/*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t *src) { @@ -2524,10 +3565,11 @@ static xxh_u64 XXH_read64(const void *memPtr) { #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 1)) /* - * __pack instructions are safer, but compiler specific, hence potentially - * problematic for some compilers. - * - * Currently only defined for GCC and ICC. + * __attribute__((aligned(1))) is supported by gcc and clang. Originally + * the documentation claimed that it only increased the alignment, but + * actually it can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. */ #ifdef XXH_OLD_NAMES typedef union { @@ -2540,14 +3582,8 @@ typedef union { #endif static xxh_u64 XXH_read64(const void *ptr) { - typedef union { - - xxh_u32 u32; - xxh_u64 u64; - - } __attribute__((packed)) xxh_unalign64; - - return ((const xxh_unalign64 *)ptr)->u64; + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64 *)ptr); } @@ -2555,12 +3591,13 @@ static xxh_u64 XXH_read64(const void *ptr) { /* * Portable and safe solution. Generally efficient. - * see: https://stackoverflow.com/a/32095106/646947 + * see: + * https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html */ static xxh_u64 XXH_read64(const void *memPtr) { xxh_u64 val; - memcpy(&val, memPtr, sizeof(val)); + XXH_memcpy(&val, memPtr, sizeof(val)); return val; } @@ -2639,8 +3676,10 @@ XXH_FORCE_INLINE xxh_u64 XXH_readLE64_align(const void *ptr, /******* xxh64 *******/ /*! * @} - * @defgroup xxh64_impl XXH64 implementation + * @defgroup XXH64_impl XXH64 implementation * @ingroup impl + * + * Details on the XXH64 implementation. * @{ */ @@ -2674,6 +3713,7 @@ XXH_FORCE_INLINE xxh_u64 XXH_readLE64_align(const void *ptr, #define PRIME64_5 XXH_PRIME64_5 #endif +/*! @copydoc XXH32_round */ static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) { acc += input * XXH_PRIME64_2; @@ -2692,51 +3732,68 @@ static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) { } -static xxh_u64 XXH64_avalanche(xxh_u64 h64) { +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) { - h64 ^= h64 >> 33; - h64 *= XXH_PRIME64_2; - h64 ^= h64 >> 29; - h64 *= XXH_PRIME64_3; - h64 ^= h64 >> 32; - return h64; + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; } #define XXH_get64bits(p) XXH_readLE64_align(p, align) -static xxh_u64 XXH64_finalize(xxh_u64 h64, const xxh_u8 *ptr, size_t len, - XXH_alignment align) { +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 XXH64_finalize(xxh_u64 hash, const xxh_u8 *ptr, + size_t len, XXH_alignment align) { + if (ptr == NULL) XXH_ASSERT(len == 0); len &= 31; while (len >= 8) { xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); ptr += 8; - h64 ^= k1; - h64 = XXH_rotl64(h64, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; + hash ^= k1; + hash = XXH_rotl64(hash, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; len -= 8; } if (len >= 4) { - h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; ptr += 4; - h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; len -= 4; } while (len > 0) { - h64 ^= (*ptr++) * XXH_PRIME64_5; - h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; --len; } - return XXH64_avalanche(h64); + return XXH64_avalanche(hash); } @@ -2750,26 +3807,25 @@ static xxh_u64 XXH64_finalize(xxh_u64 h64, const xxh_u8 *ptr, size_t len, #undef XXH_PROCESS8_64 #endif -XXH_FORCE_INLINE xxh_u64 XXH64_endian_align(const xxh_u8 *input, size_t len, - xxh_u64 seed, XXH_alignment align) { - - const xxh_u8 *bEnd = input ? input + len : NULL; - xxh_u64 h64; - - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) - if (input == NULL) { - - len = 0; - bEnd = input = (const xxh_u8 *)(size_t)32; - - } +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 XXH64_endian_align(const xxh_u8 *input, + size_t len, xxh_u64 seed, + XXH_alignment align) { - #endif + xxh_u64 h64; + if (input == NULL) XXH_ASSERT(len == 0); if (len >= 32) { - const xxh_u8 *const limit = bEnd - 32; + const xxh_u8 *const bEnd = input + len; + const xxh_u8 *const limit = bEnd - 31; xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; xxh_u64 v2 = seed + XXH_PRIME64_2; xxh_u64 v3 = seed + 0; @@ -2786,7 +3842,7 @@ XXH_FORCE_INLINE xxh_u64 XXH64_endian_align(const xxh_u8 *input, size_t len, v4 = XXH64_round(v4, XXH_get64bits(input)); input += 8; - } while (input <= limit); + } while (input < limit); h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); @@ -2807,16 +3863,17 @@ XXH_FORCE_INLINE xxh_u64 XXH64_endian_align(const xxh_u8 *input, size_t len, } -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t len, +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64(XXH_NOESCAPE const void *input, size_t len, XXH64_hash_t seed) { - #if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_state_t state; - XXH64_reset(&state, seed); - XXH64_update(&state, (const xxh_u8*)input, len); - return XXH64_digest(&state); + #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small + * inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8 *)input, len); + return XXH64_digest(&state); #else if (XXH_FORCE_ALIGN_CHECK) { @@ -2834,16 +3891,16 @@ XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t len, } -/******* Hash Streaming *******/ - -/*! @ingroup xxh64_family*/ + /******* Hash Streaming *******/ + #ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ XXH_PUBLIC_API XXH64_state_t *XXH64_createState(void) { return (XXH64_state_t *)XXH_malloc(sizeof(XXH64_state_t)); } -/*! @ingroup xxh64_family */ +/*! @ingroup XXH64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr) { XXH_free(statePtr); @@ -2851,42 +3908,39 @@ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr) { } -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t *dstState, - const XXH64_state_t *srcState) { +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t *dstState, + const XXH64_state_t *srcState) { - memcpy(dstState, srcState, sizeof(*dstState)); + XXH_memcpy(dstState, srcState, sizeof(*dstState)); } -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t *statePtr, - XXH64_hash_t seed) { +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t *statePtr, + XXH64_hash_t seed) { - XXH64_state_t state; /* use a local state to memcpy() in order to avoid - strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; - state.v2 = seed + XXH_PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - XXH_PRIME64_1; - /* do not write into reserved64, might be removed in a future version */ - memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved64)); + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; return XXH_OK; } -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *state, - const void *input, size_t len) { +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH_NOESCAPE XXH64_state_t *state, + XXH_NOESCAPE const void *input, + size_t len) { - if (input == NULL) - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) + if (input == NULL) { + + XXH_ASSERT(len == 0); return XXH_OK; - #else - return XXH_ERROR; - #endif + + } { @@ -2905,10 +3959,10 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *state, if (state->memsize) { /* tmp buffer is full */ XXH_memcpy(((xxh_u8 *)state->mem64) + state->memsize, input, 32 - state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64 + 0)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64 + 1)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64 + 2)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64 + 3)); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64 + 0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64 + 1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64 + 2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64 + 3)); p += 32 - state->memsize; state->memsize = 0; @@ -2917,29 +3971,20 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *state, if (p + 32 <= bEnd) { const xxh_u8 *const limit = bEnd - 32; - xxh_u64 v1 = state->v1; - xxh_u64 v2 = state->v2; - xxh_u64 v3 = state->v3; - xxh_u64 v4 = state->v4; do { - v1 = XXH64_round(v1, XXH_readLE64(p)); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p += 8; - v2 = XXH64_round(v2, XXH_readLE64(p)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p += 8; - v3 = XXH64_round(v3, XXH_readLE64(p)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p += 8; - v4 = XXH64_round(v4, XXH_readLE64(p)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p += 8; } while (p <= limit); - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } if (p < bEnd) { @@ -2955,28 +4000,24 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *state, } -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t *state) { +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH64_digest(XXH_NOESCAPE const XXH64_state_t *state) { xxh_u64 h64; if (state->total_len >= 32) { - xxh_u64 const v1 = state->v1; - xxh_u64 const v2 = state->v2; - xxh_u64 const v3 = state->v3; - xxh_u64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + - XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); } else { - h64 = state->v3 /*seed*/ + XXH_PRIME64_5; + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; } @@ -2987,21 +4028,23 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t *state) { } + #endif /* !XXH_NO_STREAM */ + /******* Canonical representation *******/ -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, - XXH64_hash_t hash) { +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t *dst, + XXH64_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - memcpy(dst, &hash, sizeof(*dst)); + XXH_memcpy(dst, &hash, sizeof(*dst)); } -/*! @ingroup xxh64_family */ +/*! @ingroup XXH64_family */ XXH_PUBLIC_API XXH64_hash_t -XXH64_hashFromCanonical(const XXH64_canonical_t *src) { +XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t *src) { return XXH_readBE64(src); @@ -3015,7 +4058,7 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src) { ************************************************************************ */ /*! * @} - * @defgroup xxh3_impl XXH3 implementation + * @defgroup XXH3_impl XXH3 implementation * @ingroup impl * @{ @@ -3030,9 +4073,16 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src) { #elif defined(__STDC_VERSION__) && \ __STDC_VERSION__ >= 199901L /* >= C99 */ #define XXH_RESTRICT restrict + #elif (defined(__GNUC__) && \ + ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) || \ + (defined(__clang__)) || (defined(_MSC_VER) && (_MSC_VER >= 1400)) || \ + (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) + /* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ + #define XXH_RESTRICT __restrict #else - /* Note: it might be useful to define __restrict or __restrict__ for - * some C++ compilers */ #define XXH_RESTRICT /* disable */ #endif @@ -3046,17 +4096,38 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src) { #define XXH_unlikely(x) (x) #endif - #if defined(__GNUC__) - #if defined(__AVX2__) - #include <immintrin.h> - #elif defined(__SSE2__) - #include <emmintrin.h> - #elif defined(__ARM_NEON__) || defined(__ARM_NEON) + #ifndef XXH_HAS_INCLUDE + #ifdef __has_include + /* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ + #define XXH_HAS_INCLUDE __has_include + #else + #define XXH_HAS_INCLUDE(x) 0 + #endif + #endif + + #if defined(__GNUC__) || defined(__clang__) + #if defined(__ARM_FEATURE_SVE) + #include <arm_sve.h> + #endif + #if defined(__ARM_NEON__) || defined(__ARM_NEON) || \ + (defined(_M_ARM) && _M_ARM >= 7) || defined(_M_ARM64) || \ + defined(_M_ARM64EC) || \ + (defined(__wasm_simd128__) && \ + XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */ #define inline __inline__ /* circumvent a clang bug */ #include <arm_neon.h> #undef inline + #elif defined(__AVX2__) + #include <immintrin.h> + #elif defined(__SSE2__) + #include <emmintrin.h> #endif - #elif defined(_MSC_VER) + #endif + + #if defined(_MSC_VER) #include <intrin.h> #endif @@ -3161,7 +4232,7 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src) { * Note that these are actually implemented as macros. * * If this is not defined, it is detected automatically. - * @ref XXH_X86DISPATCH overrides this. + * internal macro XXH_X86DISPATCH overrides this. */ enum XXH_VECTOR_TYPE /* fake enum */ { @@ -3174,8 +4245,13 @@ enum XXH_VECTOR_TYPE /* fake enum */ { */ XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ - XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ }; @@ -3183,7 +4259,7 @@ enum XXH_VECTOR_TYPE /* fake enum */ { * @ingroup tuning * @brief Selects the minimum alignment for XXH3's accumulators. * - * When using SIMD, this should match the alignment reqired for said + * When using SIMD, this should match the alignment required for said * vector type, so, for example, 32 for AVX2. * * Default: Auto detected. @@ -3199,23 +4275,30 @@ enum XXH_VECTOR_TYPE /* fake enum */ { #define XXH_AVX512 3 #define XXH_NEON 4 #define XXH_VSX 5 + #define XXH_SVE 6 #endif #ifndef XXH_VECTOR /* can be defined on command line */ - #if defined(__AVX512F__) + #if defined(__ARM_FEATURE_SVE) + #define XXH_VECTOR XXH_SVE + #elif (defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || \ + defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && \ + XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \ + ) && \ + (defined(_WIN32) || \ + defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + #define XXH_VECTOR XXH_NEON + #elif defined(__AVX512F__) #define XXH_VECTOR XXH_AVX512 #elif defined(__AVX2__) #define XXH_VECTOR XXH_AVX2 #elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || \ (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) #define XXH_VECTOR XXH_SSE2 - #elif defined(__GNUC__) /* msvc support maybe later */ \ - && (defined(__ARM_NEON__) || defined(__ARM_NEON)) && \ - (defined( \ - __LITTLE_ENDIAN__) /* We only support little endian NEON */ \ - || (defined(__BYTE_ORDER__) && \ - __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) - #define XXH_VECTOR XXH_NEON #elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) || \ (defined(__s390x__) && defined(__VEC__)) && \ defined(__GNUC__) /* TODO: IBM XL */ @@ -3225,6 +4308,17 @@ enum XXH_VECTOR_TYPE /* fake enum */ { #endif #endif + /* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ + #if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) + #ifdef _MSC_VER + #pragma warning(once : 4606) + #else + #warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." + #endif + #undef XXH_VECTOR + #define XXH_VECTOR XXH_SCALAR + #endif + /* * Controls the alignment of the accumulator, * for compatibility with aligned vector loads, which are usually faster. @@ -3244,16 +4338,26 @@ enum XXH_VECTOR_TYPE /* fake enum */ { #define XXH_ACC_ALIGN 16 #elif XXH_VECTOR == XXH_AVX512 /* avx512 */ #define XXH_ACC_ALIGN 64 + #elif XXH_VECTOR == XXH_SVE /* sve */ + #define XXH_ACC_ALIGN 64 #endif #endif #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 || \ XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 #define XXH_SEC_ALIGN XXH_ACC_ALIGN + #elif XXH_VECTOR == XXH_SVE + #define XXH_SEC_ALIGN XXH_ACC_ALIGN #else #define XXH_SEC_ALIGN 8 #endif + #if defined(__GNUC__) || defined(__clang__) + #define XXH_ALIASING __attribute__((may_alias)) + #else + #define XXH_ALIASING /* nothing */ + #endif + /* * UGLY HACK: * GCC usually generates the best code with -O3 for xxHash. @@ -3278,126 +4382,153 @@ enum XXH_VECTOR_TYPE /* fake enum */ { #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ && defined(__OPTIMIZE__) && \ - !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ + XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ #pragma GCC push_options #pragma GCC optimize("-O2") #endif #if XXH_VECTOR == XXH_NEON - /* - * NEON's setup for vmlal_u32 is a little more complicated than it is on - * SSE2, AVX2, and VSX. - * - * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an - * upcast. - * - * To do the same operation, the 128-bit 'Q' register needs to be split - * into two 64-bit 'D' registers, performing this operation:: - * - * [ a | b ] | - * '---------. .--------' | | x | - * | .---------' '--------. | - * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 - * ] - * - * Due to significant changes in aarch64, the fastest method for aarch64 - * is completely different than the fastest method for ARMv7-A. - * - * ARMv7-A treats D registers as unions overlaying Q registers, so - * modifying D11 will modify the high half of Q5. This is similar to how - * modifying AH will only affect bits 8-15 of AX on x86. - * - * VZIP takes two registers, and puts even lanes in one register and odd - * lanes in the other. + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC + * -O3 optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + + /*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. * - * On ARMv7-A, this strangely modifies both parameters in place instead - * of taking the usual 3-operand form. + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is + * only *conditionally* safe (`vld1` has an alignment bit like + * `movdq[ua]` in x86). * - * Therefore, if we want to do this, we can simply use a D-form VZIP.32 - * on the lower and upper halves of the Q register to end up with the - * high and low halves where we want - all in one instruction. + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so + * it prohibits load-store optimizations. Therefore, a direct + * dereference is used. * - * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a + * safe unaligned load. + */ + #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t +XXH_vld1q_u64(void const *ptr) /* silence -Wcast-align */ +{ - * d10[1], d11[1] } + return *(xxh_aliasing_uint64x2_t const *)ptr; + +} + + #else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const *ptr) { + + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const *)ptr)); + +} + + #endif + + /*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. * - * Unfortunately we need inline assembly for this: Instructions - * modifying two registers at once is not possible in GCC or Clang's IR, - * and they have to create a copy. + * This is a workaround for AArch64 GCC < 11 which implemented + * arm_neon.h with inline assembly and were therefore incapable of + * merging the `vget_{low, high}_u32` with `vmlal_u32`. + */ + #if defined(__aarch64__) && defined(__GNUC__) && \ + !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, + uint32x4_t rhs) { + + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w"(acc) : "w"(lhs), "w"(rhs)); + return acc; + +} + +XXH_FORCE_INLINE uint64x2_t XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, + uint32x4_t rhs) { + + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); + +} + + #else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, + uint32x4_t rhs) { + + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); + +} + +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, + uint32x4_t rhs) { + + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); + +} + + #endif + + /*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 * - * aarch64 requires a different approach. + * This can be set to 2, 4, 6, or 8. * - * In order to make it easier to write a decent compiler for aarch64, - * many quirks were removed, such as conditional execution. + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. * - * NEON was also affected by this. + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but + * only 2 of those can be NEON. If you are only using NEON instructions, + * you are only using 2/3 of the CPU bandwidth. * - * aarch64 cannot access the high bits of a Q-form register, and writes - * to a D-form register zero the high bits, similar to how writes to - * W-form scalar registers (or DWORD registers on x86_64) work. + * This is even more noticeable on the more advanced cores like the + * Cortex-A76 which can dispatch 8 micro-ops per cycle, but still only 2 + * NEON micro-ops at once. * - * The formerly free vget_high intrinsics now require a vext (with a few - * exceptions) + * Therefore, to make the most out of the pipeline, it is beneficial to + * run 6 NEON lanes and 2 scalar lanes, which is chosen by default. * - * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the - * equivalent of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to - * only modify one operand. + * This does not apply to Apple processors or 32-bit processors, which + * run better with full NEON. These will default to 8. Additionally, + * size-optimized builds run 8 lanes. * - * The equivalent of the VZIP.32 on the lower and upper halves would be - * this mess: + * This change benefits CPUs with large micro-op buffers without + * negatively affecting most other CPUs: * - * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], - * v0[1] } zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } zip2 - * v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * | Chipset | Dispatch type | NEON only | 6:2 + * hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 + * GB/s | ~16% | | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 + * GB/s | 5.3 GB/s | ~5% | | Marvell PXA1928 (A53) | In-order + * dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | | Apple M1 | 4 NEON/8 + * micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | * - * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 - * (SHRN): + * It also seems to fix some bad codegen on GCC, making it almost as + * fast as clang. * - * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); - * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of + * the lanes meaning it effectively becomes worse 4. * - * This is available on ARMv7-A, but is less efficient than a single - * VZIP.32. + * @see XXH3_accumulate_512_neon() */ - - /*! - * Function-like macro: - * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t - * &outHi) - * { - - * outLo = (uint32x2_t)(in & 0xFFFFFFFF); - * outHi = (uint32x2_t)(in >> 32); - * in = UNDEFINED; - * } - */ - #if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ - && defined(__GNUC__) && !defined(__aarch64__) && \ - !defined(__arm64__) - #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - \ - /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, \ - * %f0 = upper D half */ \ - /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 \ - */ \ - /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 \ - */ \ - __asm__("vzip.32 %e0, %f0" : "+w"(in)); \ - (outLo) = vget_low_u32(vreinterpretq_u32_u64(in)); \ - (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ - \ - } while (0) - - #else - #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - \ - (outLo) = vmovn_u64(in); \ - (outHi) = vshrn_n_u64((in), 32); \ - \ - } while (0) - + #ifndef XXH3_NEON_LANES + #if (defined(__aarch64__) || defined(__arm64__) || \ + defined(_M_ARM64) || defined(_M_ARM64EC)) && \ + !defined(__APPLE__) && XXH_SIZE_OPT <= 0 + #define XXH3_NEON_LANES 6 + #else + #define XXH3_NEON_LANES XXH_ACC_NB + #endif #endif #endif /* XXH_VECTOR == XXH_NEON */ @@ -3410,28 +4541,43 @@ enum XXH_VECTOR_TYPE /* fake enum */ { * inconsistent intrinsics, spotty coverage, and multiple endiannesses. */ #if XXH_VECTOR == XXH_VSX + /* Annoyingly, these headers _may_ define three macros: `bool`, + * `vector`, and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ + #pragma push_macro("bool") + #pragma push_macro("vector") + #pragma push_macro("pixel") + /* silence potential macro redefined warnings */ + #undef bool + #undef vector + #undef pixel + #if defined(__s390x__) #include <s390intrin.h> #else - /* gcc's altivec.h can have the unwanted consequence to - * unconditionally #define bool, vector, and pixel keywords, with bad - * consequences for programs already using these keywords for other - * purposes. The paragraph defining these macros is skipped when - * __APPLE_ALTIVEC__ is defined. - * __APPLE_ALTIVEC__ is _generally_ defined automatically by the - * compiler, but it seems that, in some cases, it isn't. Force the - * build macro to be defined, so that keywords are not altered. - */ - #if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) - #define __APPLE_ALTIVEC__ - #endif #include <altivec.h> #endif + /* Restore the original macro values, if applicable. */ + #pragma pop_macro("pixel") + #pragma pop_macro("vector") + #pragma pop_macro("bool") + typedef __vector unsigned long long xxh_u64x2; typedef __vector unsigned char xxh_u8x16; typedef __vector unsigned xxh_u32x4; +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing + * issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + #ifndef XXH_VSX_BE #if defined(__BIG_ENDIAN__) || \ (defined(__BYTE_ORDER__) && \ @@ -3472,7 +4618,7 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) { XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) { xxh_u64x2 ret; - memcpy(&ret, ptr, sizeof(xxh_u64x2)); + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); #if XXH_VSX_BE ret = XXH_vec_revb(ret); #endif @@ -3491,9 +4637,12 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) { /* s390x is always big endian, no issue on this platform */ #define XXH_vec_mulo vec_mulo #define XXH_vec_mule vec_mule - #elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) - /* Clang has a better way to control this, we can just use the builtin - * which doesn't swap. */ + #elif defined(__clang__) && \ + XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) + /* Clang has a better way to control this, we can just use the builtin + * which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the + * vec_* operations */ #define XXH_vec_mulo __builtin_altivec_vmulouw #define XXH_vec_mule __builtin_altivec_vmuleuw #else @@ -3519,14 +4668,33 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) { #endif /* XXH_vec_mulo, XXH_vec_mule */ #endif /* XXH_VECTOR == XXH_VSX */ + #if XXH_VECTOR == XXH_SVE + #define ACCRND(acc, offset) \ + do { \ + \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ + \ + } while (0) + + #endif /* XXH_VECTOR == XXH_SVE */ + /* prefetch * can be disabled, by declaring XXH_NO_PREFETCH build macro */ #if defined(XXH_NO_PREFETCH) #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ #else - #if defined(_MSC_VER) && \ - (defined(_M_X64) || \ - defined( \ + #if XXH_SIZE_OPT >= 1 + #define XXH_PREFETCH(ptr) (void)(ptr) + #elif defined(_MSC_VER) && \ + (defined(_M_X64) || \ + defined( \ _M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ #include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ #define XXH_PREFETCH(ptr) \ @@ -3573,6 +4741,15 @@ static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { }; +static const xxh_u64 PRIME_MX1 = + 0x165667919E3779F9ULL; /*!< + 0b0001011001010110011001111001000110011110001101110111100111111001 + */ +static const xxh_u64 PRIME_MX2 = + 0x9FB21C651E98DF25ULL; /*!< + 0b1001111110110010000111000110010100011110100110001101111100100101 + */ + #ifdef XXH_OLD_NAMES #define kSecret XXH3_kSecret #endif @@ -3601,18 +4778,17 @@ XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { } #elif defined(_MSC_VER) && defined(_M_IX86) - #include <intrin.h> #define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) #else - /* - * Downcast + upcast is usually better than masking on older compilers - * like GCC 4.2 (especially 32-bit ones), all without affecting newer - * compilers. - * - * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both - * operands and perform a full 64x64 multiply -- entirely redundant on - * 32-bit. - */ + /* + * Downcast + upcast is usually better than masking on older compilers + * like GCC 4.2 (especially 32-bit ones), all without affecting newer + * compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both + * operands and perform a full 64x64 multiply -- entirely redundant on + * 32-bit. + */ #define XXH_mult32to64(x, y) \ ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) #endif @@ -3623,7 +4799,7 @@ XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar * version. * - * @param lhs, rhs The 64-bit integers to be multiplied + * @param lhs , rhs The 64-bit integers to be multiplied * @return The 128-bit result represented in an @ref XXH128_hash_t. */ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { @@ -3643,8 +4819,8 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { * In that case it is best to use the portable one. * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 */ - #if defined(__GNUC__) && !defined(__wasm__) && \ - defined(__SIZEOF_INT128__) || \ + #if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) && \ + defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; @@ -3661,7 +4837,7 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { * * This compiles to single operand MUL on x64. */ - #elif defined(_M_X64) || defined(_M_IA64) + #elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) #ifndef _MSC_VER #pragma intrinsic(_umul128) @@ -3673,6 +4849,22 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { r128.high64 = product_high; return r128; + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t + * method. + */ + #elif defined(_M_ARM64) || defined(_M_ARM64EC) + + #ifndef _MSC_VER + #pragma intrinsic(__umulh) + #endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + #else /* * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. @@ -3744,7 +4936,7 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { * around by value. This will hopefully inline the multiply, but we don't force * it. * - * @param lhs, rhs The 64-bit integers to multiply + * @param lhs , rhs The 64-bit integers to multiply * @return The low 64 bits of the product XOR'd by the high 64 bits. * @see XXH_mult64to128() */ @@ -3756,7 +4948,7 @@ static xxh_u64 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { } /*! Seems to produce slightly better code on GCC for some reason. */ -XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) { +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) { XXH_ASSERT(0 <= shift && shift < 64); return v64 ^ (v64 >> shift); @@ -3770,7 +4962,7 @@ XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) { static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) { h64 = XXH_xorshift64(h64, 37); - h64 *= 0x165667919E3779F9ULL; + h64 *= PRIME_MX1; h64 = XXH_xorshift64(h64, 32); return h64; @@ -3785,9 +4977,9 @@ static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) { /* this mix is inspired by Pelle Evensen's rrmxmx */ h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); - h64 *= 0x9FB21C651E98DF25ULL; + h64 *= PRIME_MX2; h64 ^= (h64 >> 35) + len; - h64 *= 0x9FB21C651E98DF25ULL; + h64 *= PRIME_MX2; return XXH_xorshift64(h64, 28); } @@ -3826,9 +5018,10 @@ static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) { * * This adds an extra layer of strength for custom secrets. */ -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_1to3_64b(const xxh_u8 *input, size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_1to3_64b(const xxh_u8 *input, + size_t len, + const xxh_u8 *secret, + XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(1 <= len && len <= 3); @@ -3854,9 +5047,10 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_1to3_64b(const xxh_u8 *input, size_t len, } -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_4to8_64b(const xxh_u8 *input, size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_4to8_64b(const xxh_u8 *input, + size_t len, + const xxh_u8 *secret, + XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); @@ -3876,10 +5070,10 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_4to8_64b(const xxh_u8 *input, size_t len, } -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_9to16_64b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_9to16_64b(const xxh_u8 *input, + size_t len, + const xxh_u8 *secret, + XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); @@ -3900,10 +5094,10 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_9to16_64b(const xxh_u8 *input, } -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_0to16_64b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_0to16_64b(const xxh_u8 *input, + size_t len, + const xxh_u8 *secret, + XXH64_hash_t seed) { XXH_ASSERT(len <= 16); { @@ -3983,7 +5177,7 @@ XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8 *XXH_RESTRICT input, } /* For mid range keys, XXH3 uses a Mum-hash variant. */ -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_17to128_64b( +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_17to128_64b( const xxh_u8 *XXH_RESTRICT input, size_t len, const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { @@ -3994,6 +5188,18 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_17to128_64b( { xxh_u64 acc = len * XXH_PRIME64_1; + #if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + + acc += XXH3_mix16B(input + 16 * i, secret + 32 * i, seed); + acc += + XXH3_mix16B(input + len - 16 * (i + 1), secret + 32 * i + 16, seed); + + } while (i-- != 0); + + #else if (len > 32) { if (len > 64) { @@ -4017,16 +5223,19 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_17to128_64b( acc += XXH3_mix16B(input + 0, secret + 0, seed); acc += XXH3_mix16B(input + len - 16, secret + 16, seed); - + #endif return XXH3_avalanche(acc); } } + /*! + * @brief Maximum size of "short" key in bytes. + */ #define XXH3_MIDSIZE_MAX 240 -XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( +XXH_NO_INLINE XXH_PUREF XXH64_hash_t XXH3_len_129to240_64b( const xxh_u8 *XXH_RESTRICT input, size_t len, const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { @@ -4039,17 +5248,23 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( { - xxh_u64 acc = len * XXH_PRIME64_1; - int const nbRounds = (int)len / 16; - int i; + xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); for (i = 0; i < 8; i++) { acc += XXH3_mix16B(input + (16 * i), secret + (16 * i), seed); } - acc = XXH3_avalanche(acc); + /* last bytes */ + acc_end = XXH3_mix16B( + input + len - 16, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); #if defined(__clang__) /* Clang */ \ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ @@ -4077,17 +5292,18 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( #endif for (i = 8; i < nbRounds; i++) { - acc += + /* + * Prevents clang for unrolling the acc loop and interleaving with this + * one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input + (16 * i), secret + (16 * (i - 8)) + XXH3_MIDSIZE_STARTOFFSET, seed); } - /* last bytes */ - acc += XXH3_mix16B(input + len - 16, - secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, - seed); - return XXH3_avalanche(acc); + return XXH3_avalanche(acc + acc_end); } @@ -4105,10 +5321,49 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( #define ACC_NB XXH_ACC_NB #endif + #ifndef XXH_PREFETCH_DIST + #ifdef __clang__ + #define XXH_PREFETCH_DIST 320 + #else + #if (XXH_VECTOR == XXH_AVX512) + #define XXH_PREFETCH_DIST 512 + #else + #define XXH_PREFETCH_DIST 384 + #endif + #endif /* __clang__ */ + #endif /* XXH_PREFETCH_DIST */ + + /* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_<name>() and it calls + * XXH3_accumulate_512_<name>(). + * + * It may be useful to hand implement this function if the compiler fails + * to optimize the inline function. + */ + #define XXH3_ACCUMULATE_TEMPLATE(name) \ + void XXH3_accumulate_##name( \ + xxh_u64 *XXH_RESTRICT acc, const xxh_u8 *XXH_RESTRICT input, \ + const xxh_u8 *XXH_RESTRICT secret, size_t nbStripes) { \ + \ + size_t n; \ + for (n = 0; n < nbStripes; n++) { \ + \ + const xxh_u8 *const in = input + n * XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name(acc, in, \ + secret + n * XXH_SECRET_CONSUME_RATE); \ + \ + } \ + \ + } + XXH_FORCE_INLINE void XXH_writeLE64(void *dst, xxh_u64 v64) { if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); - memcpy(dst, &v64, sizeof(v64)); + XXH_memcpy(dst, &v64, sizeof(v64)); } @@ -4176,8 +5431,7 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_accumulate_512_avx512( /* data_key = data_vec ^ key_vec; */ __m512i const data_key = _mm512_xor_si512(data_vec, key_vec); /* data_key_lo = data_key >> 32; */ - __m512i const data_key_lo = - _mm512_shuffle_epi32(data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const data_key_lo = _mm512_srli_epi64(data_key, 32); /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ __m512i const product = _mm512_mul_epu32(data_key, data_key_lo); /* xacc[0] += swap(data_vec); */ @@ -4191,31 +5445,34 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_accumulate_512_avx512( } -/* - * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. - * - * Multiplication isn't perfect, as explained by Google in HighwayHash: - * - * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to - * // varying degrees. In descending order of goodness, bytes - * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. - * // As expected, the upper and lower bytes are much worse. - * - * Source: - * https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 - * - * Since our algorithm uses a pseudorandom secret to add some variance into the - * mix, we don't need to (or want to) mix as often or as much as HighwayHash - * does. - * - * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid - * extraction. - * - * Both XXH3_64bits and XXH3_128bits use this subroutine. - */ +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) -XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_scrambleAcc_avx512( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT secret) { + /* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: + * https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into + * the mix, we don't need to (or want to) mix as often or as much as + * HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to + * avoid extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + + XXH_FORCE_INLINE XXH_TARGET_AVX512 + void XXH3_scrambleAcc_avx512(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 63) == 0); XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); @@ -4227,14 +5484,13 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_scrambleAcc_avx512( /* xacc[0] ^= (xacc[0] >> 47) */ __m512i const acc_vec = *xacc; __m512i const shifted = _mm512_srli_epi64(acc_vec, 47); - __m512i const data_vec = _mm512_xor_si512(acc_vec, shifted); /* xacc[0] ^= secret; */ __m512i const key_vec = _mm512_loadu_si512(secret); - __m512i const data_key = _mm512_xor_si512(data_vec, key_vec); + __m512i const data_key = _mm512_ternarylogic_epi32( + key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); /* xacc[0] *= XXH_PRIME32_1; */ - __m512i const data_key_hi = - _mm512_shuffle_epi32(data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const data_key_hi = _mm512_srli_epi64(data_key, 32); __m512i const prod_lo = _mm512_mul_epu32(data_key, prime32); __m512i const prod_hi = _mm512_mul_epu32(data_key_hi, prime32); *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); @@ -4253,8 +5509,9 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512( { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); - __m512i const seed = _mm512_mask_set1_epi64( - _mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = + _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); const __m512i *const src = (const __m512i *)((const void *)XXH3_kSecret); __m512i *const dest = (__m512i *)customSecret; @@ -4263,18 +5520,7 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512( XXH_ASSERT(((size_t)dest & 63) == 0); for (i = 0; i < nbRounds; ++i) { - /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void - * const*', this will warn "discards 'const' qualifier". */ - union { - - const __m512i *cp; - void *p; - - } remote_const_void; - - remote_const_void.cp = src + i; - dest[i] = - _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); } @@ -4317,8 +5563,7 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2( /* data_key = data_vec ^ key_vec; */ __m256i const data_key = _mm256_xor_si256(data_vec, key_vec); /* data_key_lo = data_key >> 32; */ - __m256i const data_key_lo = - _mm256_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const data_key_lo = _mm256_srli_epi64(data_key, 32); /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ __m256i const product = _mm256_mul_epu32(data_key, data_key_lo); /* xacc[i] += swap(data_vec); */ @@ -4334,8 +5579,11 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2( } -XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_scrambleAcc_avx2( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT secret) { +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + + XXH_FORCE_INLINE XXH_TARGET_AVX2 + void XXH3_scrambleAcc_avx2(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 31) == 0); { @@ -4358,8 +5606,7 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_scrambleAcc_avx2( __m256i const data_key = _mm256_xor_si256(data_vec, key_vec); /* xacc[i] *= XXH_PRIME32_1; */ - __m256i const data_key_hi = - _mm256_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const data_key_hi = _mm256_srli_epi64(data_key, 32); __m256i const prod_lo = _mm256_mul_epu32(data_key, prime32); __m256i const prod_hi = _mm256_mul_epu32(data_key_hi, prime32); xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); @@ -4399,12 +5646,12 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2( XXH_ASSERT(((size_t)dest & 31) == 0); /* GCC -O2 need unroll loop manually */ - dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src + 0), seed); - dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src + 1), seed); - dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src + 2), seed); - dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src + 3), seed); - dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src + 4), seed); - dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src + 5), seed); + dest[0] = _mm256_add_epi64(_mm256_load_si256(src + 0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src + 1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src + 2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src + 3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src + 4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src + 5), seed); } @@ -4462,8 +5709,11 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_accumulate_512_sse2( } -XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_scrambleAcc_sse2( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT secret) { +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + + XXH_FORCE_INLINE XXH_TARGET_SSE2 + void XXH3_scrambleAcc_sse2(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { @@ -4546,40 +5796,173 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2( #if (XXH_VECTOR == XXH_NEON) +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void XXH3_scalarRound(void *XXH_RESTRICT acc, + void const *XXH_RESTRICT input, + void const *XXH_RESTRICT secret, + size_t lane); + +XXH_FORCE_INLINE void XXH3_scalarScrambleRound(void *XXH_RESTRICT acc, + void const *XXH_RESTRICT secret, + size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this + * optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics + * overlap nearly perfectly. + */ + XXH_FORCE_INLINE void XXH3_accumulate_512_neon( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, const void *XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); - { - - uint64x2_t *const xacc = (uint64x2_t *)acc; + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && + XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t *const xacc = (xxh_aliasing_uint64x2_t *)acc; /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ - uint8_t const *const xinput = (const uint8_t *)input; - uint8_t const *const xsecret = (const uint8_t *)secret; + uint8_t const *xinput = (const uint8_t *)input; + uint8_t const *xsecret = (const uint8_t *)secret; size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { + #ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); + #endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + + XXH3_scalarRound(acc, input, secret, i); + + } + + i = 0; + /* 4 NEON lanes at a time. */ + for (; i + 1 < XXH3_NEON_LANES / 2; i += 2) { /* data_vec = xinput[i]; */ - uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i + 1) * 16)); /* key_vec = xsecret[i]; */ - uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); - uint64x2_t data_key; - uint32x2_t data_key_lo, data_key_hi; - /* xacc[i] += swap(data_vec); */ - uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); - uint64x2_t const swapped = vextq_u64(data64, data64, 1); - xacc[i] = vaddq_u64(xacc[i], swapped); + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i + 1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); /* data_key = data_vec ^ key_vec; */ - data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); - /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); - * data_key_hi = (uint32x2_t) (data_key >> 32); - * data_key = UNDEFINED; */ - XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); - /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ - xacc[i] = vmlal_u32(xacc[i], data_key_lo, data_key_hi); + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32(vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2)); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for + * most widening intrinsics, have a variant that works on both high half + * vectors for free on AArch64. A similar instruction is available on + * SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = + XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = + XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i + 1] = vaddq_u64(xacc[i + 1], sum_2); + + } + + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum); } @@ -4587,19 +5970,37 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_neon( } -XXH_FORCE_INLINE void XXH3_scrambleAcc_neon(void *XXH_RESTRICT acc, - const void *XXH_RESTRICT secret) { +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + + XXH_FORCE_INLINE + void XXH3_scrambleAcc_neon(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { - uint64x2_t *xacc = (uint64x2_t *)acc; - uint8_t const *xsecret = (uint8_t const *)secret; - uint32x2_t prime = vdup_n_u32(XXH_PRIME32_1); + xxh_aliasing_uint64x2_t *xacc = (xxh_aliasing_uint64x2_t *)acc; + uint8_t const *xsecret = (uint8_t const *)secret; size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { + /* WASM uses operator overloads and doesn't need these. */ + #ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = + vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); + #endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + + XXH3_scalarScrambleRound(acc, secret, i); + + } + + for (i = 0; i < XXH3_NEON_LANES / 2; i++) { /* xacc[i] ^= (xacc[i] >> 47); */ uint64x2_t acc_vec = xacc[i]; @@ -4607,40 +6008,32 @@ XXH_FORCE_INLINE void XXH3_scrambleAcc_neon(void *XXH_RESTRICT acc, uint64x2_t data_vec = veorq_u64(acc_vec, shifted); /* xacc[i] ^= xsecret[i]; */ - uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); - uint64x2_t data_key = veorq_u64(data_vec, vreinterpretq_u64_u8(key_vec)); - - /* xacc[i] *= XXH_PRIME32_1 */ - uint32x2_t data_key_lo, data_key_hi; - /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); - * data_key_hi = (uint32x2_t) (xacc[i] >> 32); - * xacc[i] = UNDEFINED; */ - XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); - { /* - * prod_hi = (data_key >> 32) * XXH_PRIME32_1; - * - * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will - * incorrectly "optimize" this: - * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); - * shifted = vshll_n_u32(tmp, 32); - * to this: - * tmp = "vmulq_u64"(a, b); // no such thing! - * shifted = vshlq_n_u64(tmp, 32); - * - * However, unlike SSE, Clang lacks a 64-bit multiply routine - * for NEON, and it scalarizes two 64-bit multiplies instead. - * - * vmull_u32 has the same timing as vmul_u32, and it avoids - * this bug completely. - * See https://bugs.llvm.org/show_bug.cgi?id=39967 - */ - uint64x2_t prod_hi = vmull_u32(data_key_hi, prime); - /* xacc[i] = prod_hi << 32; */ - xacc[i] = vshlq_n_u64(prod_hi, 32); - /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ - xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); - - } + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ + #ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and + * scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; + #else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, + * reinterpreting the vector as a uint32x4_t and multiplying by { 0, + * prime, 0, prime } to cancel out the unwanted bits and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32(vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = + vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); + #endif } @@ -4656,47 +6049,54 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_vsx(void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, const void *XXH_RESTRICT secret) { - xxh_u64x2 *const xacc = (xxh_u64x2 *)acc; /* presumed aligned */ - xxh_u64x2 const *const xinput = - (xxh_u64x2 const *)input; /* no alignment restriction */ - xxh_u64x2 const *const xsecret = - (xxh_u64x2 const *)secret; /* no alignment restriction */ + /* presumed aligned */ + xxh_aliasing_u64x2 *const xacc = (xxh_aliasing_u64x2 *)acc; + xxh_u8 const *const xinput = + (xxh_u8 const *)input; /* no alignment restriction */ + xxh_u8 const *const xsecret = + (xxh_u8 const *)secret; /* no alignment restriction */ xxh_u64x2 const v32 = {32, 32}; size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { /* data_vec = xinput[i]; */ - xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16 * i); /* key_vec = xsecret[i]; */ - xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16 * i); xxh_u64x2 const data_key = data_vec ^ key_vec; /* shuffled = (data_key << 32) | (data_key >> 32); */ xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & * 0xFFFFFFFF); */ xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); - xacc[i] += product; + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; /* swap high and low halves */ #ifdef __s390x__ - xacc[i] += vec_permi(data_vec, data_vec, 2); + acc_vec += vec_permi(data_vec, data_vec, 2); #else - xacc[i] += vec_xxpermdi(data_vec, data_vec, 2); + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); #endif + xacc[i] = acc_vec; } } -XXH_FORCE_INLINE void XXH3_scrambleAcc_vsx(void *XXH_RESTRICT acc, - const void *XXH_RESTRICT secret) { +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + + XXH_FORCE_INLINE + void XXH3_scrambleAcc_vsx(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { - xxh_u64x2 *const xacc = (xxh_u64x2 *)acc; - const xxh_u64x2 *const xsecret = (const xxh_u64x2 *)secret; + xxh_aliasing_u64x2 *const xacc = (xxh_aliasing_u64x2 *)acc; + const xxh_u8 *const xsecret = (const xxh_u8 *)secret; /* constants */ xxh_u64x2 const v32 = {32, 32}; xxh_u64x2 const v47 = {47, 47}; @@ -4710,7 +6110,7 @@ XXH_FORCE_INLINE void XXH3_scrambleAcc_vsx(void *XXH_RESTRICT acc, xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); /* xacc[i] ^= xsecret[i]; */ - xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16 * i); xxh_u64x2 const data_key = data_vec ^ key_vec; /* xacc[i] *= XXH_PRIME32_1 */ @@ -4729,46 +6129,272 @@ XXH_FORCE_INLINE void XXH3_scrambleAcc_vsx(void *XXH_RESTRICT acc, #endif -/* scalar variants - universal */ + #if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void XXH3_accumulate_512_sve(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT input, + const void *XXH_RESTRICT secret) { + + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + + } else if (element_count == 2) { /* sve128 */ + + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + + } else { + + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + + } + +} + +XXH_FORCE_INLINE void XXH3_accumulate_sve(xxh_u64 *XXH_RESTRICT acc, + const xxh_u8 *XXH_RESTRICT input, + const xxh_u8 *XXH_RESTRICT secret, + size_t nbStripes) { + + if (nbStripes != 0) { + + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + + } else if (element_count == 2) { /* sve128 */ + + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + + } else { + + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + + } + + } + +} + + #endif + + /* scalar variants - universal */ + + #if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, + * only big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) + * which does not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, + xxh_u64 acc) { + + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" + : "=r"(ret) + : "r"(lhs), "r"(rhs), "r"(acc)); + return ret; + +} + + #else +XXH_FORCE_INLINE xxh_u64 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, + xxh_u64 acc) { + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; + +} + + #endif + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a + * combination of NEON and scalar. + */ +XXH_FORCE_INLINE void XXH3_scalarRound(void *XXH_RESTRICT acc, + void const *XXH_RESTRICT input, + void const *XXH_RESTRICT secret, + size_t lane) { + + xxh_u64 *xacc = (xxh_u64 *)acc; + xxh_u8 const *xinput = (xxh_u8 const *)input; + xxh_u8 const *xsecret = (xxh_u8 const *)secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN - 1)) == 0); + { + + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, + data_key >> 32, xacc[lane]); + + } + +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ XXH_FORCE_INLINE void XXH3_accumulate_512_scalar( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, const void *XXH_RESTRICT secret) { - xxh_u64 *const xacc = (xxh_u64 *)acc; /* presumed aligned */ - const xxh_u8 *const xinput = - (const xxh_u8 *)input; /* no alignment restriction */ - const xxh_u8 *const xsecret = - (const xxh_u8 *)secret; /* no alignment restriction */ size_t i; - XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN - 1)) == 0); + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on + * ARMv6. */ + #if defined(__GNUC__) && !defined(__clang__) && \ + (defined(__arm__) || defined(__thumb2__)) && \ + defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes \ + bytes */ \ + && XXH_SIZE_OPT <= 0 + #pragma GCC unroll 8 + #endif for (i = 0; i < XXH_ACC_NB; i++) { - xxh_u64 const data_val = XXH_readLE64(xinput + 8 * i); - xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i * 8); - xacc[i ^ 1] += data_val; /* swap adjacent lanes */ - xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); + XXH3_scalarRound(acc, input, secret, i); } } -XXH_FORCE_INLINE void XXH3_scrambleAcc_scalar(void *XXH_RESTRICT acc, - const void *XXH_RESTRICT secret) { +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) + + /*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a + * combination of NEON and scalar. + */ + XXH_FORCE_INLINE + void XXH3_scalarScrambleRound(void *XXH_RESTRICT acc, + void const *XXH_RESTRICT secret, + size_t lane) { xxh_u64 *const xacc = (xxh_u64 *)acc; /* presumed aligned */ const xxh_u8 *const xsecret = (const xxh_u8 *)secret; /* no alignment restriction */ - size_t i; XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN - 1)) == 0); - for (i = 0; i < XXH_ACC_NB; i++) { + XXH_ASSERT(lane < XXH_ACC_NB); + { - xxh_u64 const key64 = XXH_readLE64(xsecret + 8 * i); - xxh_u64 acc64 = xacc[i]; + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; acc64 = XXH_xorshift64(acc64, 47); acc64 ^= key64; acc64 *= XXH_PRIME32_1; - xacc[i] = acc64; + xacc[lane] = acc64; + + } + +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void XXH3_scrambleAcc_scalar(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { + + size_t i; + for (i = 0; i < XXH_ACC_NB; i++) { + + XXH3_scalarScrambleRound(acc, secret, i); } @@ -4785,15 +6411,16 @@ XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( const xxh_u8 *kSecretPtr = XXH3_kSecret; XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); - #if defined(__clang__) && defined(__aarch64__) + #if defined(__GNUC__) && defined(__aarch64__) /* * UGLY HACK: - * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are * placed sequentially, in order, at the top of the unrolled loop. * * While MOVK is great for generating constants (2 cycles for a 64-bit - * constant compared to 4 cycles for LDR), long MOVK chains stall the - * integer pipelines: + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * * I L S * MOVK * MOVK @@ -4802,7 +6429,7 @@ XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( * ADD * SUB STR * STR - * By forcing loads from memory (as the asm line causes Clang to assume + * By forcing loads from memory (as the asm line causes the compiler to assume * that XXH3_kSecretPtr has been changed), the pipelines are used more * efficiently: * I L S @@ -4810,18 +6437,15 @@ XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( * ADD LDR * SUB STR * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * * XXH3_64bits_withSeed, len == 256, Snapdragon 835 * without hack: 2654.4 MB/s * with hack: 3202.9 MB/s */ XXH_COMPILER_GUARD(kSecretPtr); #endif - /* - * Note: in debug mode, this overrides the asm optimization - * and Clang will emit MOVK chains again. - */ - XXH_ASSERT(kSecretPtr == XXH3_kSecret); - { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; @@ -4829,7 +6453,7 @@ XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( for (i = 0; i < nbRounds; i++) { /* - * The asm hack causes Clang to assume that kSecretPtr aliases with + * The asm hack causes the compiler to assume that kSecretPtr aliases with * customSecret, and on aarch64, this prevented LDP from merging two * loads together for free. Putting the loads together before the stores * properly generates LDP. @@ -4845,87 +6469,71 @@ XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( } -typedef void (*XXH3_f_accumulate_512)(void *XXH_RESTRICT, const void *, - const void *); +typedef void (*XXH3_f_accumulate)(xxh_u64 *XXH_RESTRICT, + const xxh_u8 *XXH_RESTRICT, + const xxh_u8 *XXH_RESTRICT, size_t); typedef void (*XXH3_f_scrambleAcc)(void *XXH_RESTRICT, const void *); typedef void (*XXH3_f_initCustomSecret)(void *XXH_RESTRICT, xxh_u64); #if (XXH_VECTOR == XXH_AVX512) #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 + #define XXH3_accumulate XXH3_accumulate_avx512 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 #elif (XXH_VECTOR == XXH_AVX2) #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 + #define XXH3_accumulate XXH3_accumulate_avx2 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 #elif (XXH_VECTOR == XXH_SSE2) #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 + #define XXH3_accumulate XXH3_accumulate_sse2 #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 #elif (XXH_VECTOR == XXH_NEON) #define XXH3_accumulate_512 XXH3_accumulate_512_neon + #define XXH3_accumulate XXH3_accumulate_neon #define XXH3_scrambleAcc XXH3_scrambleAcc_neon #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #elif (XXH_VECTOR == XXH_VSX) #define XXH3_accumulate_512 XXH3_accumulate_512_vsx + #define XXH3_accumulate XXH3_accumulate_vsx #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #elif (XXH_VECTOR == XXH_SVE) + #define XXH3_accumulate_512 XXH3_accumulate_512_sve + #define XXH3_accumulate XXH3_accumulate_sve + #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #else /* scalar */ #define XXH3_accumulate_512 XXH3_accumulate_512_scalar + #define XXH3_accumulate XXH3_accumulate_scalar #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #endif - #ifndef XXH_PREFETCH_DIST - #ifdef __clang__ - #define XXH_PREFETCH_DIST 320 - #else - #if (XXH_VECTOR == XXH_AVX512) - #define XXH_PREFETCH_DIST 512 - #else - #define XXH_PREFETCH_DIST 384 - #endif - #endif /* __clang__ */ - #endif /* XXH_PREFETCH_DIST */ - -/* - * XXH3_accumulate() - * Loops over XXH3_accumulate_512(). - * Assumption: nbStripes will not overflow the secret size - */ -XXH_FORCE_INLINE void XXH3_accumulate(xxh_u64 *XXH_RESTRICT acc, - const xxh_u8 *XXH_RESTRICT input, - const xxh_u8 *XXH_RESTRICT secret, - size_t nbStripes, - XXH3_f_accumulate_512 f_acc512) { - - size_t n; - for (n = 0; n < nbStripes; n++) { - - const xxh_u8 *const in = input + n * XXH_STRIPE_LEN; - XXH_PREFETCH(in + XXH_PREFETCH_DIST); - f_acc512(acc, in, secret + n * XXH_SECRET_CONSUME_RATE); - - } - -} + #if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ + #undef XXH3_initCustomSecret + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #endif XXH_FORCE_INLINE void XXH3_hashLong_internal_loop( xxh_u64 *XXH_RESTRICT acc, const xxh_u8 *XXH_RESTRICT input, size_t len, const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { + XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; @@ -4938,8 +6546,7 @@ XXH_FORCE_INLINE void XXH3_hashLong_internal_loop( for (n = 0; n < nb_blocks; n++) { - XXH3_accumulate(acc, input + n * block_len, secret, nbStripesPerBlock, - f_acc512); + f_acc(acc, input + n * block_len, secret, nbStripesPerBlock); f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); } @@ -4951,8 +6558,7 @@ XXH_FORCE_INLINE void XXH3_hashLong_internal_loop( size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); - XXH3_accumulate(acc, input + nb_blocks * block_len, secret, nbStripes, - f_acc512); + f_acc(acc, input + nb_blocks * block_len, secret, nbStripes); /* last stripe */ { @@ -4961,8 +6567,9 @@ XXH_FORCE_INLINE void XXH3_hashLong_internal_loop( #define XXH_SECRET_LASTACC_START \ 7 /* not aligned on 8, last secret is different from acc & scrambler \ */ - f_acc512(acc, p, - secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + XXH3_accumulate_512( + acc, p, + secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); } @@ -5019,13 +6626,12 @@ static XXH64_hash_t XXH3_mergeAccs(const xxh_u64 *XXH_RESTRICT acc, XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal( const void *XXH_RESTRICT input, size_t len, const void *XXH_RESTRICT secret, - size_t secretSize, XXH3_f_accumulate_512 f_acc512, - XXH3_f_scrambleAcc f_scramble) { + size_t secretSize, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; XXH3_hashLong_internal_loop(acc, (const xxh_u8 *)input, len, - (const xxh_u8 *)secret, secretSize, f_acc512, + (const xxh_u8 *)secret, secretSize, f_acc, f_scramble); /* converge into final hash */ @@ -5041,26 +6647,30 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal( } /* - * It's important for performance that XXH3_hashLong is not inlined. + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using + * AVX instruction set. When the secret size is unknown, or on GCC 12 where the + * mix of NO_INLINE and FORCE_INLINE breaks -Og, this is XXH_NO_INLINE. */ -XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_withSecret( +XXH3_WITH_SECRET_INLINE XXH64_hash_t XXH3_hashLong_64b_withSecret( const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const xxh_u8 *XXH_RESTRICT secret, size_t secretLen) { (void)seed64; return XXH3_hashLong_64b_internal(input, len, secret, secretLen, - XXH3_accumulate_512, XXH3_scrambleAcc); + XXH3_accumulate, XXH3_scrambleAcc); } /* - * It's important for performance that XXH3_hashLong is not inlined. - * Since the function is not inlined, the compiler may not be able to understand - * that, in some scenarios, its `secret` argument is actually a compile time - * constant. This variant enforces that the compiler can detect that, and uses - * this opportunity to streamline the generated code for better performance. + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction + * cache. Note that inside this no_inline function, we do inline the internal + * loop, and provide a statically defined secret size to allow optimization of + * vector loop. */ -XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_default( +XXH_NO_INLINE XXH_PUREF XXH64_hash_t XXH3_hashLong_64b_default( const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const xxh_u8 *XXH_RESTRICT secret, size_t secretLen) { @@ -5068,7 +6678,7 @@ XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_default( (void)secret; (void)secretLen; return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, - sizeof(XXH3_kSecret), XXH3_accumulate_512, + sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); } @@ -5085,19 +6695,20 @@ XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_default( * why (uop cache maybe?), but the difference is large and easily measurable. */ XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed_internal( - const void *input, size_t len, XXH64_hash_t seed, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble, - XXH3_f_initCustomSecret f_initSec) { + const void *input, size_t len, XXH64_hash_t seed, XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, XXH3_f_initCustomSecret f_initSec) { + #if XXH_SIZE_OPT <= 0 if (seed == 0) - return XXH3_hashLong_64b_internal( - input, len, XXH3_kSecret, sizeof(XXH3_kSecret), f_acc512, f_scramble); + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, + sizeof(XXH3_kSecret), f_acc, f_scramble); + #endif { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; f_initSec(secret, seed); - return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), - f_acc512, f_scramble); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), f_acc, + f_scramble); } @@ -5106,17 +6717,15 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed_internal( /* * It's important for performance that XXH3_hashLong is not inlined. */ -XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed(const void *input, - size_t len, - XXH64_hash_t seed, - const xxh_u8 *secret, - size_t secretLen) { +XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed( + const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed, + const xxh_u8 *XXH_RESTRICT secret, size_t secretLen) { (void)secret; (void)secretLen; - return XXH3_hashLong_64b_withSeed_internal( - input, len, seed, XXH3_accumulate_512, XXH3_scrambleAcc, - XXH3_initCustomSecret); + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, XXH3_accumulate, + XXH3_scrambleAcc, + XXH3_initCustomSecret); } @@ -5152,36 +6761,49 @@ XXH3_64bits_internal(const void *XXH_RESTRICT input, size_t len, /* === Public entry point === */ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *input, size_t len) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void *input, + size_t length) { - return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), - XXH3_hashLong_64b_default); + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, + sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *input, - size_t len, - const void *secret, - size_t secretSize) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void *input, size_t length, + XXH_NOESCAPE const void *secret, size_t secretSize) { - return XXH3_64bits_internal(input, len, 0, secret, secretSize, + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void *input, size_t len, +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void *input, + size_t length, XXH64_hash_t seed) { - return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); } -/* === XXH3 streaming === */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecretandSeed( + XXH_NOESCAPE const void *input, size_t length, + XXH_NOESCAPE const void *secret, size_t secretSize, XXH64_hash_t seed) { + + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, + sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, + (const xxh_u8 *)secret, secretSize); +} + + /* === XXH3 streaming === */ + #ifndef XXH_NO_STREAM /* * Malloc's a pointer that is always aligned to align. * @@ -5205,7 +6827,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void *input, size_t len, * * Align must be a power of 2 and 8 <= align <= 128. */ -static void *XXH_alignedMalloc(size_t s, size_t align) { +static XXH_MALLOCF void *XXH_alignedMalloc(size_t s, size_t align) { XXH_ASSERT(align <= 128 && align >= 8); /* range check */ XXH_ASSERT((align & (align - 1)) == 0); /* power of 2 */ @@ -5257,7 +6879,17 @@ static void XXH_alignedFree(void *p) { } -/*! @ingroup xxh3_family */ +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + * + * @see @ref streaming_example "Streaming Example" + */ XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void) { XXH3_state_t *const state = @@ -5268,7 +6900,19 @@ XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void) { } -/*! @ingroup xxh3_family */ +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref + * XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + * + * @see @ref streaming_example "Streaming Example" + */ XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr) { XXH_alignedFree(statePtr); @@ -5276,11 +6920,11 @@ XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr) { } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t *dst_state, - const XXH3_state_t *src_state) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t *dst_state, + XXH_NOESCAPE const XXH3_state_t *src_state) { - memcpy(dst_state, src_state, sizeof(*dst_state)); + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); } @@ -5303,6 +6947,7 @@ static void XXH3_reset_internal(XXH3_state_t *statePtr, XXH64_hash_t seed, statePtr->acc[6] = XXH_PRIME64_5; statePtr->acc[7] = XXH_PRIME32_1; statePtr->seed = seed; + statePtr->useSeed = (seed != 0); statePtr->extSecret = (const unsigned char *)secret; XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; @@ -5310,8 +6955,9 @@ static void XXH3_reset_internal(XXH3_state_t *statePtr, XXH64_hash_t seed, } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t *statePtr) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t *statePtr) { if (statePtr == NULL) return XXH_ERROR; XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); @@ -5319,9 +6965,10 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t *statePtr) { } -/*! @ingroup xxh3_family */ +/*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret( - XXH3_state_t *statePtr, const void *secret, size_t secretSize) { + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize) { if (statePtr == NULL) return XXH_ERROR; XXH3_reset_internal(statePtr, 0, secret, secretSize); @@ -5331,84 +6978,140 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret( } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t *statePtr, - XXH64_hash_t seed) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH64_hash_t seed) { if (statePtr == NULL) return XXH_ERROR; if (seed == 0) return XXH3_64bits_reset(statePtr); - if (seed != statePtr->seed) + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) XXH3_initCustomSecret(statePtr->customSecret, seed); XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } -/* Note : when XXH3_consumeStripes() is invoked, - * there must be a guarantee that at least one more byte must be consumed from - * input - * so that the function can blindly consume all stripes using the "normal" - * secret segment */ -XXH_FORCE_INLINE void XXH3_consumeStripes( +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecretandSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed64) { + + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; + +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a + * block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in + * the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 *XXH3_consumeStripes( xxh_u64 *XXH_RESTRICT acc, size_t *XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, const xxh_u8 *XXH_RESTRICT input, size_t nbStripes, const xxh_u8 *XXH_RESTRICT secret, size_t secretLimit, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { - - XXH_ASSERT(nbStripes <= - nbStripesPerBlock); /* can handle max 1 scramble per invocation */ - XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); - if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { - - /* need a scrambling operation */ - size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; - size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; - XXH3_accumulate(acc, input, - secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, - nbStripesToEndofBlock, f_acc512); - f_scramble(acc, secret + secretLimit); - XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, - nbStripesAfterBlock, f_acc512); - *nbStripesSoFarPtr = nbStripesAfterBlock; + XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { - } else { + const xxh_u8 *initialSecret = + secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + + } while (nbStripes >= nbStripesPerBlock); + + *nbStripesSoFarPtr = 0; + + } + + /* Process a partial block */ + if (nbStripes > 0) { - XXH3_accumulate(acc, input, - secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, - nbStripes, f_acc512); + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; *nbStripesSoFarPtr += nbStripes; } + /* Return end pointer */ + return input; + } + #ifndef XXH3_STREAM_USE_STACK + #if XXH_SIZE_OPT <= 0 && \ + !defined( \ + __clang__) /* clang doesn't need additional stack space */ + #define XXH3_STREAM_USE_STACK 1 + #endif + #endif /* * Both XXH3_64bits_update and XXH3_128bits_update use this routine. */ -XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, - const xxh_u8 *input, size_t len, - XXH3_f_accumulate_512 f_acc512, - XXH3_f_scrambleAcc f_scramble) { +XXH_FORCE_INLINE XXH_errorcode XXH3_update( + XXH3_state_t *XXH_RESTRICT const state, const xxh_u8 *XXH_RESTRICT input, + size_t len, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { - if (input == NULL) - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) + if (input == NULL) { + + XXH_ASSERT(len == 0); return XXH_OK; - #else - return XXH_ERROR; - #endif + } + + XXH_ASSERT(state != NULL); { const xxh_u8 *const bEnd = input + len; const unsigned char *const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; - + #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); + #else + xxh_u64 *XXH_RESTRICT const acc = state->acc; + #endif state->totalLen += len; XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); - if (state->bufferedSize + len <= - XXH3_INTERNALBUFFER_SIZE) { /* fill in tmp buffer */ + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); state->bufferedSize += (XXH32_hash_t)len; return XXH_OK; @@ -5416,9 +7119,8 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, } /* total input is now > XXH3_INTERNALBUFFER_SIZE */ - - #define XXH3_INTERNALBUFFER_STRIPES \ - (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + #define XXH3_INTERNALBUFFER_STRIPES \ + (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ @@ -5431,41 +7133,35 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); input += loadSize; - XXH3_consumeStripes(state->acc, &state->nbStripesSoFar, - state->nbStripesPerBlock, state->buffer, - XXH3_INTERNALBUFFER_STRIPES, secret, - state->secretLimit, f_acc512, f_scramble); + XXH3_consumeStripes(acc, &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, secret, + state->secretLimit, f_acc, f_scramble); state->bufferedSize = 0; } XXH_ASSERT(input < bEnd); - - /* Consume input by a multiple of internal buffer size */ if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { - const xxh_u8 *const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; - do { - - XXH3_consumeStripes(state->acc, &state->nbStripesSoFar, - state->nbStripesPerBlock, input, - XXH3_INTERNALBUFFER_STRIPES, secret, - state->secretLimit, f_acc512, f_scramble); - input += XXH3_INTERNALBUFFER_SIZE; - - } while (input < limit); - - /* for last partial stripe */ - memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, - input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes( + acc, &state->nbStripesSoFar, state->nbStripesPerBlock, input, + nbStripes, secret, state->secretLimit, f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, + input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); } - XXH_ASSERT(input < bEnd); - /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); XXH_memcpy(state->buffer, input, (size_t)(bEnd - input)); state->bufferedSize = (XXH32_hash_t)(bEnd - input); + #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); + #endif } @@ -5473,11 +7169,12 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update(XXH3_state_t *state, - const void *input, size_t len) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t *state, + XXH_NOESCAPE const void *input, size_t len) { - return XXH3_update(state, (const xxh_u8 *)input, len, XXH3_accumulate_512, + return XXH3_update(state, (const xxh_u8 *)input, len, XXH3_accumulate, XXH3_scrambleAcc); } @@ -5486,41 +7183,46 @@ XXH_FORCE_INLINE void XXH3_digest_long(XXH64_hash_t *acc, const XXH3_state_t *state, const unsigned char *secret) { + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8 *lastStripePtr; + /* * Digest on a local copy. This way, the state remains unaltered, and it can * continue ingesting more input afterwards. */ - memcpy(acc, state->acc, sizeof(state->acc)); + XXH_memcpy(acc, state->acc, sizeof(state->acc)); if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; size_t nbStripesSoFar = state->nbStripesSoFar; XXH3_consumeStripes(acc, &nbStripesSoFar, state->nbStripesPerBlock, state->buffer, nbStripes, secret, state->secretLimit, - XXH3_accumulate_512, XXH3_scrambleAcc); - /* last stripe */ - XXH3_accumulate_512(acc, - state->buffer + state->bufferedSize - XXH_STRIPE_LEN, - secret + state->secretLimit - XXH_SECRET_LASTACC_START); + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; } else { /* bufferedSize < XXH_STRIPE_LEN */ - xxh_u8 lastStripe[XXH_STRIPE_LEN]; + /* Copy to temp buffer */ size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ - memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, - catchupSize); - memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); - XXH3_accumulate_512(acc, lastStripe, - secret + state->secretLimit - XXH_SECRET_LASTACC_START); + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, + catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; } + /* Last stripe */ + XXH3_accumulate_512(acc, lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *state) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_digest(XXH_NOESCAPE const XXH3_state_t *state) { const unsigned char *const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; @@ -5534,7 +7236,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *state) { } /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ - if (state->seed) + if (state->useSeed) return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), @@ -5542,69 +7244,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *state) { } - #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) - -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API void XXH3_generateSecret(void *secretBuffer, - const void *customSeed, - size_t customSeedSize) { - - XXH_ASSERT(secretBuffer != NULL); - if (customSeedSize == 0) { - - memcpy(secretBuffer, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); - return; - - } - - XXH_ASSERT(customSeed != NULL); - - { - - size_t const segmentSize = sizeof(XXH128_hash_t); - size_t const nbSegments = XXH_SECRET_DEFAULT_SIZE / segmentSize; - XXH128_canonical_t scrambler; - XXH64_hash_t seeds[12]; - size_t segnb; - XXH_ASSERT(nbSegments == 12); - XXH_ASSERT(segmentSize * nbSegments == - XXH_SECRET_DEFAULT_SIZE); /* exact multiple */ - XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); - - /* - * Copy customSeed to seeds[], truncating or repeating as necessary. - */ - { - - size_t toFill = XXH_MIN(customSeedSize, sizeof(seeds)); - size_t filled = toFill; - memcpy(seeds, customSeed, toFill); - while (filled < sizeof(seeds)) { - - toFill = XXH_MIN(filled, sizeof(seeds) - filled); - memcpy((char *)seeds + filled, seeds, toFill); - filled += toFill; - - } - - } - - /* generate secret */ - memcpy(secretBuffer, &scrambler, sizeof(scrambler)); - for (segnb = 1; segnb < nbSegments; segnb++) { - - size_t const segmentStart = segnb * segmentSize; - XXH128_canonical_t segment; - XXH128_canonicalFromHash(&segment, - XXH128(&scrambler, sizeof(scrambler), - XXH_readLE64(seeds + segnb) + segnb)); - memcpy((char *)secretBuffer + segmentStart, &segment, sizeof(segment)); - - } - - } - -} + #endif /* !XXH_NO_STREAM */ /* ========================================== * XXH3 128 bits (a.k.a XXH128) @@ -5623,10 +7263,8 @@ XXH_PUBLIC_API void XXH3_generateSecret(void *secretBuffer, * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). */ -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_1to3_128b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_1to3_128b( + const xxh_u8 *input, size_t len, const xxh_u8 *secret, XXH64_hash_t seed) { /* A doubled version of 1to3_64b with different constants. */ XXH_ASSERT(input != NULL); @@ -5660,10 +7298,8 @@ XXH_FORCE_INLINE XXH128_hash_t XXH3_len_1to3_128b(const xxh_u8 *input, } -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_4to8_128b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_4to8_128b( + const xxh_u8 *input, size_t len, const xxh_u8 *secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); @@ -5686,7 +7322,7 @@ XXH_FORCE_INLINE XXH128_hash_t XXH3_len_4to8_128b(const xxh_u8 *input, m128.low64 ^= (m128.high64 >> 3); m128.low64 = XXH_xorshift64(m128.low64, 35); - m128.low64 *= 0x9FB21C651E98DF25ULL; + m128.low64 *= PRIME_MX2; m128.low64 = XXH_xorshift64(m128.low64, 28); m128.high64 = XXH3_avalanche(m128.high64); return m128; @@ -5695,10 +7331,8 @@ XXH_FORCE_INLINE XXH128_hash_t XXH3_len_4to8_128b(const xxh_u8 *input, } -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_9to16_128b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_9to16_128b( + const xxh_u8 *input, size_t len, const xxh_u8 *secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); @@ -5788,10 +7422,8 @@ XXH_FORCE_INLINE XXH128_hash_t XXH3_len_9to16_128b(const xxh_u8 *input, /* * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN */ -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_0to16_128b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_0to16_128b( + const xxh_u8 *input, size_t len, const xxh_u8 *secret, XXH64_hash_t seed) { XXH_ASSERT(len <= 16); { @@ -5833,7 +7465,7 @@ XXH_FORCE_INLINE XXH128_hash_t XXH128_mix32B(XXH128_hash_t acc, } -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_17to128_128b( +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_17to128_128b( const xxh_u8 *XXH_RESTRICT input, size_t len, const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { @@ -5846,6 +7478,22 @@ XXH_FORCE_INLINE XXH128_hash_t XXH3_len_17to128_128b( XXH128_hash_t acc; acc.low64 = len * XXH_PRIME64_1; acc.high64 = 0; + + #if XXH_SIZE_OPT >= 1 + { + + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + + acc = XXH128_mix32B(acc, input + 16 * i, input + len - 16 * (i + 1), + secret + 32 * i, seed); + + } while (i-- != 0); + + } + + #else if (len > 32) { if (len > 64) { @@ -5867,6 +7515,7 @@ XXH_FORCE_INLINE XXH128_hash_t XXH3_len_17to128_128b( } acc = XXH128_mix32B(acc, input, input + len - 16, secret, seed); + #endif { XXH128_hash_t h128; @@ -5883,7 +7532,7 @@ XXH_FORCE_INLINE XXH128_hash_t XXH3_len_17to128_128b( } -XXH_NO_INLINE XXH128_hash_t XXH3_len_129to240_128b( +XXH_NO_INLINE XXH_PUREF XXH128_hash_t XXH3_len_129to240_128b( const xxh_u8 *XXH_RESTRICT input, size_t len, const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { @@ -5894,25 +7543,33 @@ XXH_NO_INLINE XXH128_hash_t XXH3_len_129to240_128b( { XXH128_hash_t acc; - int const nbRounds = (int)len / 32; - int i; + unsigned i; acc.low64 = len * XXH_PRIME64_1; acc.high64 = 0; - for (i = 0; i < 4; i++) { + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { - acc = XXH128_mix32B(acc, input + (32 * i), input + (32 * i) + 16, - secret + (32 * i), seed); + acc = XXH128_mix32B(acc, input + i - 32, input + i - 16, secret + i - 32, + seed); } acc.low64 = XXH3_avalanche(acc.low64); acc.high64 = XXH3_avalanche(acc.high64); - XXH_ASSERT(nbRounds >= 4); - for (i = 4; i < nbRounds; i++) { + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i = 160; i <= len; i += 32) { - acc = XXH128_mix32B(acc, input + (32 * i), input + (32 * i) + 16, - secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), - seed); + acc = XXH128_mix32B(acc, input + i - 32, input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, seed); } @@ -5920,7 +7577,7 @@ XXH_NO_INLINE XXH128_hash_t XXH3_len_129to240_128b( acc = XXH128_mix32B( acc, input + len - 16, input + len - 32, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, - 0ULL - seed); + (XXH64_hash_t)0 - seed); { @@ -5941,12 +7598,12 @@ XXH_NO_INLINE XXH128_hash_t XXH3_len_129to240_128b( XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_internal( const void *XXH_RESTRICT input, size_t len, const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { + XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; XXH3_hashLong_internal_loop(acc, (const xxh_u8 *)input, len, secret, - secretSize, f_acc512, f_scramble); + secretSize, f_acc, f_scramble); /* converge into final hash */ XXH_STATIC_ASSERT(sizeof(acc) == 64); @@ -5966,9 +7623,9 @@ XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_internal( } /* - * It's important for performance that XXH3_hashLong is not inlined. + * It's important for performance that XXH3_hashLong() is not inlined. */ -XXH_NO_INLINE XXH128_hash_t XXH3_hashLong_128b_default( +XXH_NO_INLINE XXH_PUREF XXH128_hash_t XXH3_hashLong_128b_default( const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const void *XXH_RESTRICT secret, size_t secretLen) { @@ -5976,39 +7633,43 @@ XXH_NO_INLINE XXH128_hash_t XXH3_hashLong_128b_default( (void)secret; (void)secretLen; return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, - sizeof(XXH3_kSecret), XXH3_accumulate_512, + sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); } /* - * It's important for performance that XXH3_hashLong is not inlined. + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and + * FORCE_INLINE breaks -Og, this is XXH_NO_INLINE. */ -XXH_NO_INLINE XXH128_hash_t XXH3_hashLong_128b_withSecret( +XXH3_WITH_SECRET_INLINE XXH128_hash_t XXH3_hashLong_128b_withSecret( const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const void *XXH_RESTRICT secret, size_t secretLen) { (void)seed64; return XXH3_hashLong_128b_internal(input, len, (const xxh_u8 *)secret, - secretLen, XXH3_accumulate_512, + secretLen, XXH3_accumulate, XXH3_scrambleAcc); } XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_withSeed_internal( const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble, + XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble, XXH3_f_initCustomSecret f_initSec) { if (seed64 == 0) - return XXH3_hashLong_128b_internal( - input, len, XXH3_kSecret, sizeof(XXH3_kSecret), f_acc512, f_scramble); + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, + sizeof(XXH3_kSecret), f_acc, f_scramble); { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; f_initSec(secret, seed64); return XXH3_hashLong_128b_internal(input, len, (const xxh_u8 *)secret, - sizeof(secret), f_acc512, f_scramble); + sizeof(secret), f_acc, f_scramble); } @@ -6023,9 +7684,9 @@ XXH3_hashLong_128b_withSeed(const void *input, size_t len, XXH64_hash_t seed64, (void)secret; (void)secretLen; - return XXH3_hashLong_128b_withSeed_internal( - input, len, seed64, XXH3_accumulate_512, XXH3_scrambleAcc, - XXH3_initCustomSecret); + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, + XXH3_initCustomSecret); } @@ -6060,8 +7721,9 @@ XXH3_128bits_internal(const void *input, size_t len, XXH64_hash_t seed64, /* === Public XXH128 API === */ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void *input, size_t len) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void *input, + size_t len) { return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), @@ -6069,21 +7731,19 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void *input, size_t len) { } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *input, - size_t len, - const void *secret, - size_t secretSize) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void *input, size_t len, + XXH_NOESCAPE const void *secret, size_t secretSize) { return XXH3_128bits_internal(input, len, 0, (const xxh_u8 *)secret, secretSize, XXH3_hashLong_128b_withSecret); } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void *input, - size_t len, - XXH64_hash_t seed) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed( + XXH_NOESCAPE const void *input, size_t len, XXH64_hash_t seed) { return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), @@ -6091,67 +7751,80 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void *input, } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH128(const void *input, size_t len, +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecretandSeed( + XXH_NOESCAPE const void *input, size_t len, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed) { + + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, + sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); + +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH128(XXH_NOESCAPE const void *input, size_t len, XXH64_hash_t seed) { return XXH3_128bits_withSeed(input, len, seed); } -/* === XXH3 128-bit streaming === */ - + /* === XXH3 128-bit streaming === */ + #ifndef XXH_NO_STREAM /* - * All the functions are actually the same as for 64-bit streaming variant. - * The only difference is the finalization routine. + * All initialization and update functions are identical to 64-bit streaming + * variant. The only difference is the finalization routine. */ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t *statePtr) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t *statePtr) { - if (statePtr == NULL) return XXH_ERROR; - XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; + return XXH3_64bits_reset(statePtr); } -/*! @ingroup xxh3_family */ +/*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret( - XXH3_state_t *statePtr, const void *secret, size_t secretSize) { + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize) { - if (statePtr == NULL) return XXH_ERROR; - XXH3_reset_internal(statePtr, 0, secret, secretSize); - if (secret == NULL) return XXH_ERROR; - if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; - return XXH_OK; + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t *statePtr, - XXH64_hash_t seed) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH64_hash_t seed) { - if (statePtr == NULL) return XXH_ERROR; - if (seed == 0) return XXH3_128bits_reset(statePtr); - if (seed != statePtr->seed) - XXH3_initCustomSecret(statePtr->customSecret, seed); - XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; + return XXH3_64bits_reset_withSeed(statePtr, seed); } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t *state, - const void *input, - size_t len) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecretandSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed) { - return XXH3_update(state, (const xxh_u8 *)input, len, XXH3_accumulate_512, - XXH3_scrambleAcc); + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, + seed); } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t *state) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t *state, + XXH_NOESCAPE const void *input, size_t len) { + + return XXH3_64bits_update(state, input, len); + +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_digest(XXH_NOESCAPE const XXH3_state_t *state) { const unsigned char *const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; @@ -6186,12 +7859,13 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t *state) { } - /* 128-bit utility functions */ + #endif /* !XXH_NO_STREAM */ + /* 128-bit utility functions */ #include <string.h> /* memcmp, memcpy */ /* return : 1 is equal, 0 if different */ -/*! @ingroup xxh3_family */ +/*! @ingroup XXH3_family */ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { /* note : XXH128_hash_t is compact, it has no padding byte */ @@ -6200,11 +7874,12 @@ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { } /* This prototype is compatible with stdlib's qsort(). - * return : >0 if *h128_1 > *h128_2 - * <0 if *h128_1 < *h128_2 - * =0 if *h128_1 == *h128_2 */ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API int XXH128_cmp(const void *h128_1, const void *h128_2) { + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void *h128_1, + XXH_NOESCAPE const void *h128_2) { XXH128_hash_t const h1 = *(const XXH128_hash_t *)h128_1; XXH128_hash_t const h2 = *(const XXH128_hash_t *)h128_2; @@ -6216,9 +7891,9 @@ XXH_PUBLIC_API int XXH128_cmp(const void *h128_1, const void *h128_2) { } /*====== Canonical representation ======*/ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t *dst, - XXH128_hash_t hash) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void XXH128_canonicalFromHash( + XXH_NOESCAPE XXH128_canonical_t *dst, XXH128_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) { @@ -6228,14 +7903,15 @@ XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t *dst, } - memcpy(dst, &hash.high64, sizeof(hash.high64)); - memcpy((char *)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char *)dst + sizeof(hash.high64), &hash.low64, + sizeof(hash.low64)); } -/*! @ingroup xxh3_family */ +/*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t -XXH128_hashFromCanonical(const XXH128_canonical_t *src) { +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t *src) { XXH128_hash_t h; h.high64 = XXH_readBE64(src); @@ -6244,11 +7920,99 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src) { } + /* ========================================== + * Secret generators + * ========================================== + */ + #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void *dst, XXH128_hash_t h128) { + + XXH_writeLE64(dst, XXH_readLE64(dst) ^ h128.low64); + XXH_writeLE64((char *)dst + 8, XXH_readLE64((char *)dst + 8) ^ h128.high64); + +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret( + XXH_NOESCAPE void *secretBuffer, size_t secretSize, + XXH_NOESCAPE const void *customSeed, size_t customSeedSize) { + + #if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + #else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + #endif + + if (customSeedSize == 0) { + + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + + } + + #if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); + #else + if (customSeed == NULL) return XXH_ERROR; + #endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { + + size_t pos = 0; + while (pos < secretSize) { + + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char *)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + + } + + } + + { + + size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n = 0; n < nbSeg16; n++) { + + XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n); + XXH3_combine16((char *)secretBuffer + n * 16, h128); + + } + + /* last segment */ + XXH3_combine16((char *)secretBuffer + secretSize - 16, + XXH128_hashFromCanonical(&scrambler)); + + } + + return XXH_OK; + +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed( + XXH_NOESCAPE void *secretBuffer, XXH64_hash_t seed) { + + XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + XXH3_initCustomSecret(secret, seed); + XXH_ASSERT(secretBuffer != NULL); + memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE); + +} + /* Pop our optimization override from above */ #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ && defined(__OPTIMIZE__) && \ - !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ + XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ #pragma GCC pop_options #endif @@ -6263,7 +8027,7 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src) { #if defined(__cplusplus) -} +} /* extern "C" */ #endif diff --git a/injections.dic b/injections.dic new file mode 100644 index 00000000..4063cd17 --- /dev/null +++ b/injections.dic @@ -0,0 +1,7 @@ +"1'\" OR \"1\"=\"1" +"1\"' OR '1'='1" +"'\"><FUZZ" +"*)(FUZZ=*))(|" +"\";FUZZ;\"" +"';FUZZ;'" +"$(FUZZ)" diff --git a/instrumentation/README.injections.md b/instrumentation/README.injections.md new file mode 100644 index 00000000..16cc3713 --- /dev/null +++ b/instrumentation/README.injections.md @@ -0,0 +1,48 @@ +# Injection fuzzing + +Coverage guided fuzzing so far is only able to detect crashes, so usually +memory corruption issues, or - if implemented by hand in the harness - +invariants. + +This is a proof-of-concept implementation to additionally hunt for injection +vulnerabilities. +It works by instrumenting calls to specific functions and parsing the +query parameter for a specific unescaped dictionary string, and if detected, +crashes the target. + +This has a very low false positive rate. +But obviously this can only find injection vulnerailities that are suspectible +to this specific (but most common) issue. Hence in a rare kind of injection +vulnerability this won't find the bug - and be a false negative. +But this can be tweaked by the user - see the HOW TO MODIFY section below. + +## How to use + +Set one or more of the following environment variables for **compiling** +the target and - *this is important* - when **fuzzing** the target: + + - `AFL_LLVM_INJECTIONS_SQL` + - `AFL_LLVM_INJECTIONS_LDAP` + - `AFL_LLVM_INJECTIONS_XSS` + +Alternatively you can set `AFL_LLVM_INJECTIONS_ALL` to enable all. + +## How to modify + +If you want to add more fuctions to check for e.g. SQL injections: +Add these to `instrumentation/injection-pass.cc` and recompile. + +If you want to test for more injection inputs: +Add the dictionary tokens to `src/afl-fuzz.c` and the check for them to +`instrumentation/afl-compiler-rt.o.c`. + +If you want to add new injection targets: +You will have to edit all three files. + +Just search for: +``` +// Marker: ADD_TO_INJECTIONS +``` +in the files to see where this needs to be added. + +**NOTE:** pull requests to improve this feature are highly welcome :-) diff --git a/instrumentation/README.llvm.md b/instrumentation/README.llvm.md index 126cf1a2..34b80c85 100644 --- a/instrumentation/README.llvm.md +++ b/instrumentation/README.llvm.md @@ -7,7 +7,7 @@ For the GCC-based instrumentation, see ## 1) Introduction -! llvm_mode works with llvm versions 3.8 up to 13 ! +! llvm_mode works with llvm versions 3.8 up to 17 - but 13+ is recommended ! The code in this directory allows you to instrument programs for AFL++ using true compiler-level instrumentation, instead of the more crude assembly-level diff --git a/instrumentation/README.lto.md b/instrumentation/README.lto.md index df59cc2a..bd479c26 100644 --- a/instrumentation/README.lto.md +++ b/instrumentation/README.lto.md @@ -2,7 +2,7 @@ ## TL;DR: -This version requires a LLVM 11 or newer. +This version requires a LLVM 12 or newer. 1. Use afl-clang-lto/afl-clang-lto++ because the resulting binaries run slightly faster and give better coverage. @@ -10,7 +10,7 @@ This version requires a LLVM 11 or newer. 2. You can use it together with COMPCOV, COMPLOG and the instrument file listing features. -3. It only works with LLVM 11 or newer. +3. It only works with LLVM 12 or newer. 4. AUTODICTIONARY feature (see below) @@ -60,7 +60,7 @@ AUTODICTIONARY: 11 strings found [+] Instrumented 12071 locations with no collisions (on average 1046 collisions would be in afl-gcc/afl-clang-fast) (non-hardened mode). ``` -## Getting LLVM 11+ +## Getting LLVM 12+ ### Installing llvm @@ -73,7 +73,7 @@ chmod +x llvm.sh sudo ./llvm.sh 15 all ``` -LLVM 11 to 16 should be available in all current Linux repositories. +LLVM 12 to 18 should be available in all current Linux repositories. ## How to build afl-clang-lto @@ -277,7 +277,7 @@ AS=llvm-as ... afl-clang-lto is still work in progress. Known issues: -* Anything that LLVM 11+ cannot compile, afl-clang-lto cannot compile either - +* Anything that LLVM 12+ cannot compile, afl-clang-lto cannot compile either - obviously. * Anything that does not compile with LTO, afl-clang-lto cannot compile either - obviously. @@ -319,7 +319,7 @@ Still more problems came up though as this only works without bugs from LLVM 9 onwards, and with high optimization the link optimization ruins the instrumented control flow graph. -This is all now fixed with LLVM 11+. The llvm's own linker is now able to load +This is all now fixed with LLVM 12+. The llvm's own linker is now able to load passes and this bypasses all problems we had. Happy end :) diff --git a/instrumentation/README.persistent_mode.md b/instrumentation/README.persistent_mode.md index 14e59f4a..8e4f6ae4 100644 --- a/instrumentation/README.persistent_mode.md +++ b/instrumentation/README.persistent_mode.md @@ -195,4 +195,34 @@ Then as first line after the `__AFL_LOOP` while loop: int len = __AFL_FUZZ_TESTCASE_LEN; ``` -And that is all! \ No newline at end of file +And that is all! + +## 6) Persistent record, and replay + +If your software under test requires keeping a state between persistent loop iterations (i.e., a stateful network stack), you can use the `AFL_PERSISTENT_RECORD` variable as described in the [environment variables documentation](../docs/env_variables.md). + +When `AFL_PERSISTENT_RECORD` is enabled, replay functionality is also included in the compiler-rt library. To replay a specific record, assign the record number to the AFL_PERSISTENT_REPLAY environment variable (i.e., `RECORD:XXXXX`` -> `AFL_PERSISTENT_REPLAY=XXXXX`), and run the test binary as you would normally do. +The directory where the record files live can be specified via the `AFL_PERSISTENT_DIR` environment varilable, otherwise by default it will be considered the current directory (`./`). + +If your harness reads the input files from arguments using the special `@@` argument you will need to include support by enabling `AFL_PERSISTENT_ARGPARSE` in `config.h`. + +In order to offer transparent support to harnesses using the `@@` command line argument, arguments are parsed by the `__afl_record_replay_init` init function. Since not all systems support passing arguments to initializers, this functionality is disabled by default, it's recommendable to use the `__AFL_FUZZ_TESTCASE_BUF/__AFL_FUZZ_TESTCASE_LEN` shared memory mechanism instead. + +## 7) Drop-in persistent loop replay replacement + +To use the replay functionality without having to use `afl-cc`, include the [include/record_compat.h](../include/afl-record_compat.h) header file. Together with the [include/afl-persistent-replay.h](../include/afl-persistent-replay.h) header included in it, `afl-record-compat.h` provides a drop-in replacement for the persistent loop mechanism. + +```c +#ifndef __AFL_FUZZ_TESTCASE_LEN + // #define AFL_PERSISTENT_REPLAY_ARGPARSE + #include "afl-record-compat.h" +#endif + +__AFL_FUZZ_INIT(); +``` + +A simple example is provided in [persistent_demo_replay.c](../utils/replay_record/persistent_demo_replay.c). + +Be aware that the [afl-record-compat.h](../include/afl-record-compat.h) header should only be included in a single compilation unit, or you will end up with clobbered functions and variables. + +If you need a cleaner solution, you'll have to move the functions and variables defined in [include/record_compat.h](../include/afl-record-compat.h) and [include/afl-persistent-replay.h](../include/afl-persistent-replay.h) in a C file, and add the relevant declarations to a header file. After including the new header file, the compilation unit resulting from compiling the C file can then be linked with your program. \ No newline at end of file diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index b3b0d2cd..a09f28a9 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -192,12 +192,15 @@ class ModuleSanitizerCoverageLTO PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); private: - void instrumentFunction(Function &F, DomTreeCallback DTCallback, - PostDomTreeCallback PDTCallback); - void InjectCoverageForIndirectCalls(Function &F, - ArrayRef<Instruction *> IndirCalls); - bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks, - bool IsLeafFunc = true); + void instrumentFunction(Function &F, DomTreeCallback DTCallback, + PostDomTreeCallback PDTCallback); + /* void InjectCoverageForIndirectCalls(Function &F, + ArrayRef<Instruction *> + IndirCalls);*/ + bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks, + bool IsLeafFunc = true); + bool Fake_InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks, + bool IsLeafFunc = true); GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements, Function &F, Type *Ty, const char *Section); @@ -247,6 +250,9 @@ class ModuleSanitizerCoverageLTO uint32_t afl_global_id = 0; uint32_t unhandled = 0; uint32_t select_cnt = 0; + uint32_t instrument_ctx = 0; + uint32_t instrument_ctx_max_depth = 0; + uint32_t extra_ctx_inst = 0; uint64_t map_addr = 0; const char *skip_nozero = NULL; const char *use_threadsafe_counters = nullptr; @@ -257,11 +263,14 @@ class ModuleSanitizerCoverageLTO IntegerType *Int32Tyi = NULL; IntegerType *Int64Tyi = NULL; ConstantInt *Zero = NULL; + ConstantInt *Zero32 = NULL; ConstantInt *One = NULL; LLVMContext *Ct = NULL; Module *Mo = NULL; + GlobalVariable *AFLContext = NULL; GlobalVariable *AFLMapPtr = NULL; Value *MapPtrFixed = NULL; + AllocaInst *CTX_add = NULL; std::ofstream dFile; size_t found = 0; // AFL++ END @@ -420,16 +429,51 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( setvbuf(stdout, NULL, _IONBF, 0); if (getenv("AFL_DEBUG")) { debug = 1; } if (getenv("AFL_LLVM_DICT2FILE_NO_MAIN")) { autodictionary_no_main = 1; } + if (getenv("AFL_LLVM_CALLER") || getenv("AFL_LLVM_CTX") || + getenv("AFL_LLVM_LTO_CALLER") || getenv("AFL_LLVM_LTO_CTX")) { + + instrument_ctx = 1; + + } + + if (getenv("AFL_LLVM_LTO_CALLER_DEPTH")) { + + instrument_ctx_max_depth = atoi(getenv("AFL_LLVM_LTO_CALLER_DEPTH")); + + } else if (getenv("AFL_LLVM_LTO_CTX_DEPTH")) { + + instrument_ctx_max_depth = atoi(getenv("AFL_LLVM_LTO_CTX_DEPTH")); + + } else if (getenv("AFL_LLVM_CALLER_DEPTH")) { + + instrument_ctx_max_depth = atoi(getenv("AFL_LLVM_CALLER_DEPTH")); + + } else if (getenv("AFL_LLVM_CTX_DEPTH")) { + + instrument_ctx_max_depth = atoi(getenv("AFL_LLVM_CTX_DEPTH")); + + } if ((isatty(2) && !getenv("AFL_QUIET")) || debug) { + char buf[64] = {}; + if (instrument_ctx) { + + snprintf(buf, sizeof(buf), " (CTX mode, depth %u)\n", + instrument_ctx_max_depth); + + } + SAYF(cCYA "afl-llvm-lto" VERSION cRST - " by Marc \"vanHauser\" Heuse <mh@mh-sec.de>\n"); + "%s by Marc \"vanHauser\" Heuse <mh@mh-sec.de>\n", + buf); - } else + } else { be_quiet = 1; + } + skip_nozero = getenv("AFL_LLVM_SKIP_NEVERZERO"); use_threadsafe_counters = getenv("AFL_LLVM_THREADSAFE_INST"); @@ -442,7 +486,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( if ((ptr = getenv("AFL_LLVM_DOCUMENT_IDS")) != NULL) { dFile.open(ptr, std::ofstream::out | std::ofstream::app); - if (dFile.is_open()) WARNF("Cannot access document file %s", ptr); + if (!dFile.is_open()) WARNF("Cannot access document file %s", ptr); } @@ -500,7 +544,12 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( } + AFLContext = new GlobalVariable( + M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_ctx", 0, + GlobalVariable::GeneralDynamicTLSModel, 0, false); + Zero = ConstantInt::get(Int8Tyi, 0); + Zero32 = ConstantInt::get(Int32Tyi, 0); One = ConstantInt::get(Int8Tyi, 1); initInstrumentList(); @@ -597,12 +646,12 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( } dictionary.push_back(std::string((char *)&val, len)); - found++; + ++found; if (val2) { dictionary.push_back(std::string((char *)&val2, len)); - found++; + ++found; } @@ -692,33 +741,37 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( * prototype */ FunctionType *FT = Callee->getFunctionType(); - isStrcmp &= FT->getNumParams() == 2 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()); - isStrcasecmp &= FT->getNumParams() == 2 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()); + isStrcmp &= + FT->getNumParams() == 2 && + FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0); + isStrcasecmp &= + FT->getNumParams() == 2 && + FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0); isMemcmp &= FT->getNumParams() == 3 && FT->getReturnType()->isIntegerTy(32) && FT->getParamType(0)->isPointerTy() && FT->getParamType(1)->isPointerTy() && FT->getParamType(2)->isIntegerTy(); - isStrncmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()) && - FT->getParamType(2)->isIntegerTy(); - isStrncasecmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()) && - FT->getParamType(2)->isIntegerTy(); + isStrncmp &= + FT->getNumParams() == 3 && + FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0) && + FT->getParamType(2)->isIntegerTy(); + isStrncasecmp &= + FT->getNumParams() == 3 && + FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0) && + FT->getParamType(2)->isIntegerTy(); isStdString &= FT->getNumParams() >= 2 && FT->getParamType(0)->isPointerTy() && FT->getParamType(1)->isPointerTy(); @@ -746,12 +799,12 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( else Str2 = TmpStr.str(); - if (debug) + /*if (debug) fprintf(stderr, "F:%s %p(%s)->\"%s\"(%s) %p(%s)->\"%s\"(%s)\n", FuncName.c_str(), Str1P, Str1P->getName().str().c_str(), Str1.c_str(), HasStr1 == true ? "true" : "false", Str2P, Str2P->getName().str().c_str(), Str2.c_str(), - HasStr2 == true ? "true" : "false"); + HasStr2 == true ? "true" : "false");*/ // we handle the 2nd parameter first because of llvm memcpy if (!HasStr2) { @@ -925,7 +978,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( '\0') { thestring.append("\0", 1); // add null byte - optLen++; + ++optLen; } @@ -1076,12 +1129,12 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( for (auto token : dictionary) { memlen += token.length(); - count++; + ++count; } if (!be_quiet) - printf("AUTODICTIONARY: %lu string%s found\n", count, + printf("AUTODICTIONARY: %zu string%s found\n", count, count == 1 ? "" : "s"); if (count) { @@ -1097,7 +1150,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( ptrhld.get()[offset++] = (uint8_t)token.length(); memcpy(ptrhld.get() + offset, token.c_str(), token.length()); offset += token.length(); - count++; + ++count; } @@ -1144,7 +1197,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( WARNF("No instrumentation targets found."); else { - char modeline[100]; + char modeline[128]; snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", @@ -1152,9 +1205,16 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( getenv("AFL_USE_TSAN") ? ", TSAN" : "", getenv("AFL_USE_CFISAN") ? ", CFISAN" : "", getenv("AFL_USE_UBSAN") ? ", UBSAN" : ""); - OKF("Instrumented %u locations (%u selects) without collisions (%llu " - "collisions have been avoided) (%s mode).", - inst, select_cnt, calculateCollisions(inst), modeline); + char buf[64] = {}; + if (instrument_ctx) { + + snprintf(buf, sizeof(buf), " with %u extra map entries for CTX", + extra_ctx_inst); + + } + + OKF("Instrumented %u locations (%u selects)%s (%s mode).", inst, + select_cnt, buf, modeline); } @@ -1235,13 +1295,68 @@ static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, } +/// return the number of calls to this function +u32 countCallers(Function *F) { + + u32 callers = 0; + + if (!F) { return 0; } + + for (auto *U : F->users()) { + + if (auto *CI = dyn_cast<CallInst>(U)) { + + ++callers; + (void)(CI); + + } + + } + + return callers; + +} + +/// return the calling function of a function - only if there is a single caller +Function *returnOnlyCaller(Function *F) { + + Function *caller = NULL; + + if (!F) { return NULL; } + + for (auto *U : F->users()) { + + if (auto *CI = dyn_cast<CallInst>(U)) { + + if (caller == NULL) { + + caller = CI->getParent()->getParent(); + + } else { + + return NULL; + + } + + } + + } + + return caller; + +} + void ModuleSanitizerCoverageLTO::instrumentFunction( Function &F, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) { if (F.empty()) return; if (F.getName().find(".module_ctor") != std::string::npos) return; // Should not instrument sanitizer init functions. +#if LLVM_VERSION_MAJOR >= 18 + if (F.getName().starts_with("__sanitizer_")) +#else if (F.getName().startswith("__sanitizer_")) +#endif return; // Don't instrument __sanitizer_* callbacks. // Don't touch available_externally functions, their actual body is elsewhere. if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) return; @@ -1264,6 +1379,37 @@ void ModuleSanitizerCoverageLTO::instrumentFunction( // AFL++ START if (!F.size()) return; + + LLVMContext &Context = F.getContext(); + MDNode *N = MDNode::get(Context, MDString::get(Context, "nosanitize")); + + if (instrument_ctx) { + + // we have to set __afl_ctx 0 for all indirect calls in all functions, even + // those not to be instrumented. + for (auto &BB : F) { + + for (auto &IN : BB) { + + if (auto *Call = dyn_cast<CallInst>(&IN)) { + + if (Call->isIndirectCall()) { + + IRBuilder<> Builder(IN.getContext()); + Builder.SetInsertPoint(IN.getParent(), IN.getIterator()); + StoreInst *StoreCtx = Builder.CreateStore(Zero32, AFLContext); + StoreCtx->setMetadata("nosanitize", N); + + } + + } + + } + + } + + } + if (!isInInstrumentList(&F, FMNAME)) return; // AFL++ END @@ -1277,11 +1423,297 @@ void ModuleSanitizerCoverageLTO::instrumentFunction( const PostDominatorTree *PDT = PDTCallback(F); bool IsLeafFunc = true; uint32_t skip_next = 0; + uint32_t call_counter = 0, call_depth = 0; + uint32_t inst_save = inst, save_global = afl_global_id; + uint32_t inst_in_this_func = 0; + Function *caller = NULL; + LoadInst *PrevCtxLoad = NULL; + + CTX_add = NULL; + + if (debug) fprintf(stderr, "Function: %s\n", F.getName().str().c_str()); + + if (instrument_ctx) { + + caller = &F; + call_counter = countCallers(caller); + Function *callee = caller; + + if (call_counter == 1 && instrument_ctx_max_depth) { + + ++call_depth; + + while (instrument_ctx_max_depth >= call_depth && + ((caller = returnOnlyCaller(callee)) || 1 == 1) && + (call_counter = countCallers(callee)) == 1) { + + if (debug && caller && callee) + fprintf(stderr, "DEBUG: another depth: %s <- %s [%u]\n", + callee->getName().str().c_str(), + caller->getName().str().c_str(), call_depth); + ++call_depth; + callee = caller; + + } + + if (!caller && callee) { + + caller = callee; + if (debug) + fprintf(stderr, "DEBUG: depth found: %s <- %s [count=%u, depth=%u]\n", + caller->getName().str().c_str(), F.getName().str().c_str(), + call_counter, call_depth); + + } + + } + + if (debug && call_counter < 2) { + + fprintf(stderr, "Function %s only %u (%s)\n", F.getName().str().c_str(), + call_counter, caller->getName().str().c_str()); + + } + + if (call_counter == 1) { + + call_counter = 0; + caller = NULL; + + } + + if (debug) { + + fprintf(stderr, "DEBUG: result: Function=%s callers=%u depth=%u\n", + F.getName().str().c_str(), call_counter, call_depth); + + } + + if (call_counter > 1) { + + // Fake instrumentation so we can count how many instrumentations there + // will be in this function + for (auto &BB : F) { + + for (auto &IN : BB) { + + CallInst *callInst = nullptr; + + if ((callInst = dyn_cast<CallInst>(&IN))) { + + Function *Callee = callInst->getCalledFunction(); + if (!Callee) continue; + if (callInst->getCallingConv() != llvm::CallingConv::C) continue; + StringRef FuncName = Callee->getName(); + + if (FuncName.compare(StringRef("__afl_coverage_interesting"))) + continue; + + ++inst; + + } + + SelectInst *selectInst = nullptr; + + if ((selectInst = dyn_cast<SelectInst>(&IN))) { + + Value *condition = selectInst->getCondition(); + auto t = condition->getType(); + + if (t->getTypeID() == llvm::Type::IntegerTyID) { + + inst += 2; + + } else + +#if LLVM_VERSION_MAJOR >= 14 + if (t->getTypeID() == llvm::Type::FixedVectorTyID) { + + FixedVectorType *tt = dyn_cast<FixedVectorType>(t); + if (tt) { + + uint32_t elements = tt->getElementCount().getFixedValue(); + inst += elements * 2; + + } + + } else + +#endif + { + + continue; + + } + + } + + } + + if (shouldInstrumentBlock(F, &BB, DT, PDT, Options)) + BlocksToInstrument.push_back(&BB); + + } + + Fake_InjectCoverage(F, BlocksToInstrument, IsLeafFunc); + + if (debug) + fprintf(stderr, "DEBUG: CTX: %u instrumentations\n", inst - inst_save); + + // we only instrument functions that have more than one instrumented block + if (inst > inst_save + 1) { + + inst_in_this_func = inst - inst_save; + bool done = false; + + // in rare occasions there can be multiple entry points per function + for (auto &BB : F) { + + if (&BB == &F.getEntryBlock() && done == false) { + + // we insert a CTX value in all our callers: + IRBuilder<> Builder(Context); + CallInst *CI = NULL; + Function *F2 = NULL; + uint32_t instrumented_calls = 0; + + for (auto *U : caller->users()) { + + if ((CI = dyn_cast<CallInst>(U))) { + + F2 = CI->getParent()->getParent(); + if (debug) + fprintf(stderr, + "DEBUG: CTX call insert %s [%u/%u] -> %s/%s\n", + F2->getName().str().c_str(), instrumented_calls + 1, + call_counter, caller->getName().str().c_str(), + F.getName().str().c_str()); + + Builder.SetInsertPoint(CI); + StoreInst *StoreCtx = Builder.CreateStore( + ConstantInt::get(Type::getInt32Ty(Context), + instrumented_calls++), + AFLContext); + StoreCtx->setMetadata("nosanitize", N); + + } + + } + + if (instrumented_calls != call_counter) { + + fprintf(stderr, "BUG! %s/%s <=> %u vs %u\n", + caller->getName().str().c_str(), + F.getName().str().c_str(), instrumented_calls, + call_counter); + exit(-1); + + } + + done = true; + + } + + // in all entrypoints we have to load the CTX value + if (&BB == &F.getEntryBlock()) { + + Value *CTX_offset; + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + IRBuilder<> IRB(&(*IP)); + + PrevCtxLoad = IRB.CreateLoad( +#if LLVM_VERSION_MAJOR >= 14 + IRB.getInt32Ty(), +#endif + AFLContext); + PrevCtxLoad->setMetadata("nosanitize", N); + + CTX_offset = IRB.CreateMul( + ConstantInt::get(Type::getInt32Ty(Context), inst_in_this_func), + PrevCtxLoad, "CTXmul", false, true); + + CTX_add = + IRB.CreateAlloca(Type::getInt32Ty(Context), nullptr, "CTX_add"); + auto nosan = IRB.CreateStore(CTX_offset, CTX_add); + nosan->setMetadata("nosanitize", N); + + if (debug) + fprintf( + stderr, "DEBUG: extra CTX instrumentations for %s: %u * %u\n", + F.getName().str().c_str(), inst - inst_save, call_counter); + + } + + for (auto &IN : BB) { + + // check all calls and where callee count == 1 instrument + // our current caller_id to __afl_ctx + if (auto callInst = dyn_cast<CallInst>(&IN)) { + + Function *Callee = callInst->getCalledFunction(); + if (countCallers(Callee) == 1) { + + if (debug) + fprintf(stderr, "DEBUG: %s call to %s with only one caller\n", + F.getName().str().c_str(), + Callee->getName().str().c_str()); + + IRBuilder<> Builder(IN.getContext()); + Builder.SetInsertPoint(callInst); + StoreInst *StoreCtx = + Builder.CreateStore(PrevCtxLoad, AFLContext); + StoreCtx->setMetadata("nosanitize", N); + + } + + } + + } + + } + + } + + } + + inst = inst_save; + + /* if (debug) + fprintf(stderr, "Next instrumentation (%u-%u=%u %u-%u=%u)\n", inst, + inst_save, inst - inst_save, afl_global_id, save_global, + afl_global_id - save_global);*/ + + } for (auto &BB : F) { + skip_next = 0; + + /* + uint32_t j = 0; + fprintf(stderr, "BB %p ============================================\n", + CTX_add);*/ + for (auto &IN : BB) { + /* j++; + uint32_t i = 1; + std::string errMsg; + raw_string_ostream os(errMsg); + IN.print(os); + fprintf(stderr, "Next instruction, BB size now %zu: %02u %s\n", + BB.size(), j, os.str().c_str()); for (auto &IN2 : BB) { + + std::string errMsg2; + raw_string_ostream os2(errMsg2); + IN2.print(os2); + fprintf( + stderr, "%s %02u: %s\n", + strcmp(os.str().c_str(), os2.str().c_str()) == 0 ? ">>>" : " + ", i++, os2.str().c_str()); + + }*/ + CallInst *callInst = nullptr; if ((callInst = dyn_cast<CallInst>(&IN))) { @@ -1305,6 +1737,19 @@ void ModuleSanitizerCoverageLTO::instrumentFunction( if (FuncName.compare(StringRef("__afl_coverage_interesting"))) continue; Value *val = ConstantInt::get(Int32Ty, ++afl_global_id); + if (CTX_add) { + + IRBuilder<> Builder(Context); + LoadInst *CTX_load = Builder.CreateLoad( +#if LLVM_VERSION_MAJOR >= 14 + Builder.getInt32Ty(), +#endif + CTX_add); + ModuleSanitizerCoverageLTO::SetNoSanitizeMetadata(CTX_load); + val = Builder.CreateAdd(val, CTX_load); + + } + callInst->setOperand(1, val); ++inst; @@ -1312,164 +1757,228 @@ void ModuleSanitizerCoverageLTO::instrumentFunction( SelectInst *selectInst = nullptr; - /* - std::string errMsg; - raw_string_ostream os(errMsg); - IN.print(os); - fprintf(stderr, "X(%u): %s\n", skip_next, os.str().c_str()); - */ - if (!skip_next && (selectInst = dyn_cast<SelectInst>(&IN))) { + if ((selectInst = dyn_cast<SelectInst>(&IN))) { - uint32_t vector_cnt = 0; - Value *condition = selectInst->getCondition(); - Value *result; - auto t = condition->getType(); - IRBuilder<> IRB(selectInst->getNextNode()); + if (!skip_next) { - ++select_cnt; + // fprintf(stderr, "Select in\n"); - if (t->getTypeID() == llvm::Type::IntegerTyID) { + uint32_t vector_cnt = 0; + Value *condition = selectInst->getCondition(); + Value *result; + auto t = condition->getType(); + IRBuilder<> IRB(selectInst->getNextNode()); - Value *val1 = ConstantInt::get(Int32Ty, ++afl_global_id); - Value *val2 = ConstantInt::get(Int32Ty, ++afl_global_id); - result = IRB.CreateSelect(condition, val1, val2); - skip_next = 1; - inst += 2; + ++select_cnt; + + if (t->getTypeID() == llvm::Type::IntegerTyID) { - } else + Value *val1 = ConstantInt::get(Int32Ty, ++afl_global_id); + Value *val2 = ConstantInt::get(Int32Ty, ++afl_global_id); + if (CTX_add) { + LoadInst *CTX_load = IRB.CreateLoad( #if LLVM_VERSION_MAJOR >= 14 - if (t->getTypeID() == llvm::Type::FixedVectorTyID) { + IRB.getInt32Ty(), +#endif + CTX_add); + val1 = IRB.CreateAdd(val1, CTX_load); + val2 = IRB.CreateAdd(val2, CTX_load); - FixedVectorType *tt = dyn_cast<FixedVectorType>(t); - if (tt) { + } - uint32_t elements = tt->getElementCount().getFixedValue(); - vector_cnt = elements; - inst += vector_cnt * 2; - if (elements) { + result = IRB.CreateSelect(condition, val1, val2); + skip_next = 1; + inst += 2; - FixedVectorType *GuardPtr1 = - FixedVectorType::get(Int32Ty, elements); - FixedVectorType *GuardPtr2 = - FixedVectorType::get(Int32Ty, elements); - Value *x, *y; + } else - Value *val1 = ConstantInt::get(Int32Ty, ++afl_global_id); - Value *val2 = ConstantInt::get(Int32Ty, ++afl_global_id); - x = IRB.CreateInsertElement(GuardPtr1, val1, (uint64_t)0); - y = IRB.CreateInsertElement(GuardPtr2, val2, (uint64_t)0); +#if LLVM_VERSION_MAJOR >= 14 + if (t->getTypeID() == llvm::Type::FixedVectorTyID) { - for (uint64_t i = 1; i < elements; i++) { + FixedVectorType *tt = dyn_cast<FixedVectorType>(t); + if (tt) { - val1 = ConstantInt::get(Int32Ty, ++afl_global_id); - val2 = ConstantInt::get(Int32Ty, ++afl_global_id); - x = IRB.CreateInsertElement(GuardPtr1, val1, i); - y = IRB.CreateInsertElement(GuardPtr2, val2, i); + uint32_t elements = tt->getElementCount().getFixedValue(); + vector_cnt = elements; + inst += vector_cnt * 2; + if (elements) { - } + FixedVectorType *GuardPtr1 = + FixedVectorType::get(Int32Ty, elements); + FixedVectorType *GuardPtr2 = + FixedVectorType::get(Int32Ty, elements); + Value *x, *y; - result = IRB.CreateSelect(condition, x, y); - skip_next = 1; + Value *val1 = ConstantInt::get(Int32Ty, ++afl_global_id); + Value *val2 = ConstantInt::get(Int32Ty, ++afl_global_id); + if (CTX_add) { - } + LoadInst *CTX_load = IRB.CreateLoad( + #if LLVM_VERSION_MAJOR >= 14 + IRB.getInt32Ty(), + #endif + CTX_add); + val1 = IRB.CreateAdd(val1, CTX_load); + val2 = IRB.CreateAdd(val2, CTX_load); - } + } - } else + x = IRB.CreateInsertElement(GuardPtr1, val1, (uint64_t)0); + y = IRB.CreateInsertElement(GuardPtr2, val2, (uint64_t)0); -#endif - { + for (uint64_t i = 1; i < elements; i++) { - unhandled++; - continue; + val1 = ConstantInt::get(Int32Ty, ++afl_global_id); + val2 = ConstantInt::get(Int32Ty, ++afl_global_id); + /*if (CTX_add) { // already loaded I guess - } + LoadInst *CTX_load = IRB.CreateLoad( + #if LLVM_VERSION_MAJOR >= 14 + IRB.getInt32Ty(), + #endif + CTX_add); + val1 = IRB.CreateAdd(val1, CTX_load); + val2 = IRB.CreateAdd(val2, CTX_load); - uint32_t vector_cur = 0; - /* Load SHM pointer */ - LoadInst *MapPtr = - IRB.CreateLoad(PointerType::get(Int8Ty, 0), AFLMapPtr); - ModuleSanitizerCoverageLTO::SetNoSanitizeMetadata(MapPtr); + }*/ - while (1) { + x = IRB.CreateInsertElement(GuardPtr1, val1, i); + y = IRB.CreateInsertElement(GuardPtr2, val2, i); - /* Get CurLoc */ - Value *MapPtrIdx = nullptr; + } - /* Load counter for CurLoc */ - if (!vector_cnt) { + result = IRB.CreateSelect(condition, x, y); + skip_next = 1; - MapPtrIdx = IRB.CreateGEP(Int8Ty, MapPtr, result); + } - } else { + } - auto element = IRB.CreateExtractElement(result, vector_cur++); - MapPtrIdx = IRB.CreateGEP(Int8Ty, MapPtr, element); + } else + +#endif + { + + ++unhandled; + continue; } - if (use_threadsafe_counters) { + uint32_t vector_cur = 0; + /* Load SHM pointer */ + LoadInst *MapPtr = + IRB.CreateLoad(PointerType::get(Int8Ty, 0), AFLMapPtr); + ModuleSanitizerCoverageLTO::SetNoSanitizeMetadata(MapPtr); + + while (1) { + + /* Get CurLoc */ + Value *MapPtrIdx = nullptr; + + /* Load counter for CurLoc */ + if (!vector_cnt) { + + MapPtrIdx = IRB.CreateGEP(Int8Ty, MapPtr, result); + + } else { + + auto element = IRB.CreateExtractElement(result, vector_cur++); + MapPtrIdx = IRB.CreateGEP(Int8Ty, MapPtr, element); + + } + + if (use_threadsafe_counters) { - IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One, + IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, + One, #if LLVM_VERSION_MAJOR >= 13 - llvm::MaybeAlign(1), + llvm::MaybeAlign(1), #endif - llvm::AtomicOrdering::Monotonic); + llvm::AtomicOrdering::Monotonic); + + } else { - } else { + LoadInst *Counter = IRB.CreateLoad(IRB.getInt8Ty(), MapPtrIdx); + ModuleSanitizerCoverageLTO::SetNoSanitizeMetadata(Counter); - LoadInst *Counter = IRB.CreateLoad(IRB.getInt8Ty(), MapPtrIdx); - ModuleSanitizerCoverageLTO::SetNoSanitizeMetadata(Counter); + /* Update bitmap */ - /* Update bitmap */ + Value *Incr = IRB.CreateAdd(Counter, One); - Value *Incr = IRB.CreateAdd(Counter, One); + if (skip_nozero == NULL) { - if (skip_nozero == NULL) { + auto cf = IRB.CreateICmpEQ(Incr, Zero); + auto carry = IRB.CreateZExt(cf, Int8Ty); + Incr = IRB.CreateAdd(Incr, carry); - auto cf = IRB.CreateICmpEQ(Incr, Zero); - auto carry = IRB.CreateZExt(cf, Int8Ty); - Incr = IRB.CreateAdd(Incr, carry); + } + + auto nosan = IRB.CreateStore(Incr, MapPtrIdx); + ModuleSanitizerCoverageLTO::SetNoSanitizeMetadata(nosan); } - auto nosan = IRB.CreateStore(Incr, MapPtrIdx); - ModuleSanitizerCoverageLTO::SetNoSanitizeMetadata(nosan); + if (!vector_cnt || vector_cnt == vector_cur) { break; } } - if (!vector_cnt || vector_cnt == vector_cur) { break; } - - } + skip_next = 1; + // fprintf(stderr, "Select out\n"); - skip_next = 1; + } else { - } else { + // fprintf(stderr, "Select skip\n"); + skip_next = 0; - skip_next = 0; + } } } - if (shouldInstrumentBlock(F, &BB, DT, PDT, Options)) - BlocksToInstrument.push_back(&BB); - for (auto &Inst : BB) { + if (!instrument_ctx) + if (shouldInstrumentBlock(F, &BB, DT, PDT, Options)) + BlocksToInstrument.push_back(&BB); - if (Options.IndirectCalls) { + /* + for (auto &Inst : BB) { - CallBase *CB = dyn_cast<CallBase>(&Inst); - if (CB && !CB->getCalledFunction()) IndirCalls.push_back(&Inst); + if (Options.IndirectCalls) { - } + CallBase *CB = dyn_cast<CallBase>(&Inst); + if (CB && !CB->getCalledFunction()) IndirCalls.push_back(&Inst); - } + } + + }*/ } InjectCoverage(F, BlocksToInstrument, IsLeafFunc); - InjectCoverageForIndirectCalls(F, IndirCalls); + // InjectCoverageForIndirectCalls(F, IndirCalls); + + /*if (debug) + fprintf(stderr, "Done instrumentation (%u-%u=%u %u-%u=%u)\n", inst, + inst_save, inst - inst_save, afl_global_id, save_global, + afl_global_id - save_global);*/ + + if (inst_in_this_func && call_counter > 1) { + + if (inst_in_this_func != afl_global_id - save_global) { + + fprintf( + stderr, + "BUG! inst_in_this_func %u != afl_global_id %u - save_global %u\n", + inst_in_this_func, afl_global_id, save_global); + exit(-1); + + } + + extra_ctx_inst += inst_in_this_func * (call_counter - 1); + afl_global_id += extra_ctx_inst; + + } } @@ -1478,8 +1987,8 @@ GlobalVariable *ModuleSanitizerCoverageLTO::CreateFunctionLocalArrayInSection( ArrayType *ArrayTy = ArrayType::get(Ty, NumElements); auto Array = new GlobalVariable( - *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, - Constant::getNullValue(ArrayTy), "__sancov_gen_"); + *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, + Constant::getNullValue(ArrayTy), "__sancov_gen_"); #if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && @@ -1493,7 +2002,7 @@ GlobalVariable *ModuleSanitizerCoverageLTO::CreateFunctionLocalArrayInSection( Array->setComdat(Comdat); #endif Array->setSection(getSectionName(Section)); - Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedSize())); + Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue())); GlobalsToAppendToUsed.push_back(Array); GlobalsToAppendToCompilerUsed.push_back(Array); MDNode *MD = MDNode::get(F.getContext(), ValueAsMetadata::get(&F)); @@ -1595,6 +2104,34 @@ bool ModuleSanitizerCoverageLTO::InjectCoverage( } +bool ModuleSanitizerCoverageLTO::Fake_InjectCoverage( + Function &F, ArrayRef<BasicBlock *> AllBlocks, bool IsLeafFunc) { + + if (AllBlocks.empty()) return false; + + for (size_t i = 0, N = AllBlocks.size(); i < N; i++) { + + if (BlockList.size()) { + + int skip = 0; + for (uint32_t k = 0; k < BlockList.size(); k++) { + + if (AllBlocks[i] == BlockList[k]) { skip = 1; } + + } + + if (skip) continue; + + } + + ++inst; // InjectCoverageAtBlock() + + } + + return true; + +} + // On every indirect call we call a run-time function // __sanitizer_cov_indir_call* with two parameters: // - callee address, @@ -1602,6 +2139,7 @@ bool ModuleSanitizerCoverageLTO::InjectCoverage( // The cache is used to speed up recording the caller-callee pairs. // The address of the caller is passed implicitly via caller PC. // CacheSize is encoded in the name of the run-time function. +/* void ModuleSanitizerCoverageLTO::InjectCoverageForIndirectCalls( Function &F, ArrayRef<Instruction *> IndirCalls) { @@ -1620,6 +2158,8 @@ void ModuleSanitizerCoverageLTO::InjectCoverageForIndirectCalls( } +*/ + void ModuleSanitizerCoverageLTO::InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx, @@ -1666,6 +2206,19 @@ void ModuleSanitizerCoverageLTO::InjectCoverageAtBlock(Function &F, /* Set the ID of the inserted basic block */ ConstantInt *CurLoc = ConstantInt::get(Int32Tyi, afl_global_id); + Value *val = CurLoc; + + if (CTX_add) { + + LoadInst *CTX_load = IRB.CreateLoad( +#if LLVM_VERSION_MAJOR >= 14 + IRB.getInt32Ty(), +#endif + CTX_add); + ModuleSanitizerCoverageLTO::SetNoSanitizeMetadata(CTX_load); + val = IRB.CreateAdd(CurLoc, CTX_load); + + } /* Load SHM pointer */ @@ -1673,13 +2226,13 @@ void ModuleSanitizerCoverageLTO::InjectCoverageAtBlock(Function &F, if (map_addr) { - MapPtrIdx = IRB.CreateGEP(Int8Ty, MapPtrFixed, CurLoc); + MapPtrIdx = IRB.CreateGEP(Int8Ty, MapPtrFixed, val); } else { LoadInst *MapPtr = IRB.CreateLoad(PointerType::get(Int8Ty, 0), AFLMapPtr); ModuleSanitizerCoverageLTO::SetNoSanitizeMetadata(MapPtr); - MapPtrIdx = IRB.CreateGEP(Int8Ty, MapPtr, CurLoc); + MapPtrIdx = IRB.CreateGEP(Int8Ty, MapPtr, val); } @@ -1714,12 +2267,10 @@ void ModuleSanitizerCoverageLTO::InjectCoverageAtBlock(Function &F, // done :) - inst++; + ++inst; // AFL++ END /* - XXXXXXXXXXXXXXXXXXX - auto GuardPtr = IRB.CreateIntToPtr( IRB.CreateAdd(IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), ConstantInt::get(IntptrTy, Idx * 4)), diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 41c38283..01881f28 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -13,42 +13,63 @@ #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" -#if LLVM_VERSION_MAJOR < 17 - #include "llvm/ADT/Triple.h" - #include "llvm/Analysis/EHPersonalities.h" -#else - #include "llvm/IR/EHPersonalities.h" +#if LLVM_VERSION_MAJOR >= 15 + #if LLVM_VERSION_MAJOR < 17 + #include "llvm/ADT/Triple.h" + #endif #endif #include "llvm/Analysis/PostDominators.h" -#include "llvm/IR/CFG.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/IR/CFG.h" +#endif #include "llvm/IR/Constant.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/IR/DebugInfo.h" +#endif #include "llvm/IR/Dominators.h" +#if LLVM_VERSION_MAJOR >= 17 + #include "llvm/IR/EHPersonalities.h" +#else + #include "llvm/Analysis/EHPersonalities.h" +#endif #include "llvm/IR/Function.h" -#include "llvm/IR/GlobalVariable.h" +#if LLVM_VERSION_MAJOR >= 16 + #include "llvm/IR/GlobalVariable.h" +#endif #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/InlineAsm.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/IR/InlineAsm.h" +#endif #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/MDBuilder.h" -#include "llvm/IR/Mangler.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/IR/MDBuilder.h" + #include "llvm/IR/Mangler.h" +#endif #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/PassPlugin.h" #include "llvm/IR/Type.h" -#include "llvm/InitializePasses.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/InitializePasses.h" +#endif #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/SpecialCaseList.h" #include "llvm/Support/VirtualFileSystem.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/Instrumentation.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/Support/raw_ostream.h" +#endif +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/Instrumentation.h" +#else + #include "llvm/TargetParser/Triple.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/ModuleUtils.h" -#include "llvm/Passes/PassPlugin.h" -#include "llvm/Passes/PassBuilder.h" -#include "llvm/IR/PassManager.h" #include "config.h" #include "debug.h" @@ -58,7 +79,8 @@ using namespace llvm; #define DEBUG_TYPE "sancov" -const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir"; +static const uint64_t SanCtorAndDtorPriority = 2; + const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc"; const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1"; const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2"; @@ -68,22 +90,13 @@ const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1"; const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2"; const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4"; const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8"; -const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4"; -const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8"; -const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep"; const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch"; + const char SanCovModuleCtorTracePcGuardName[] = "sancov.module_ctor_trace_pc_guard"; -const char SanCovModuleCtor8bitCountersName[] = - "sancov.module_ctor_8bit_counters"; -const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag"; -static const uint64_t SanCtorAndDtorPriority = 2; +const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init"; const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard"; -const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init"; -const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init"; -const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init"; -const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init"; const char SanCovGuardsSectionName[] = "sancov_guards"; const char SanCovCountersSectionName[] = "sancov_cntrs"; @@ -99,27 +112,9 @@ namespace { SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) { - // Sets CoverageType and IndirectCalls. - // SanitizerCoverageOptions CLOpts = getOptions(ClCoverageLevel); - Options.CoverageType = - SanitizerCoverageOptions::SCK_Edge; // std::max(Options.CoverageType, - // CLOpts.CoverageType); - Options.IndirectCalls = false; // CLOpts.IndirectCalls; - Options.TraceCmp = false; //|= ClCMPTracing; - Options.TraceDiv = false; //|= ClDIVTracing; - Options.TraceGep = false; //|= ClGEPTracing; - Options.TracePC = false; //|= ClTracePC; - Options.TracePCGuard = true; // |= ClTracePCGuard; - Options.Inline8bitCounters = 0; //|= ClInline8bitCounters; - // Options.InlineBoolFlag = 0; //|= ClInlineBoolFlag; - Options.PCTable = false; //|= ClCreatePCTable; - Options.NoPrune = false; //|= !ClPruneBlocks; - Options.StackDepth = false; //|= ClStackDepth; - if (!Options.TracePCGuard && !Options.TracePC && - !Options.Inline8bitCounters && !Options.StackDepth /*&& - !Options.InlineBoolFlag*/) - Options.TracePCGuard = true; // TracePCGuard is default. - + Options.CoverageType = SanitizerCoverageOptions::SCK_Edge; + // Options.NoPrune = true; + Options.TracePCGuard = true; // TracePCGuard is default. return Options; } @@ -139,20 +134,13 @@ class ModuleSanitizerCoverageAFL } PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); - - bool instrumentModule(Module &M, DomTreeCallback DTCallback, - PostDomTreeCallback PDTCallback); + bool instrumentModule(Module &M, DomTreeCallback DTCallback, + PostDomTreeCallback PDTCallback); private: void instrumentFunction(Function &F, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback); - void InjectCoverageForIndirectCalls(Function &F, - ArrayRef<Instruction *> IndirCalls); void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets); - void InjectTraceForDiv(Function &F, - ArrayRef<BinaryOperator *> DivTraceTargets); - void InjectTraceForGep(Function &F, - ArrayRef<GetElementPtrInst *> GepTraceTargets); void InjectTraceForSwitch(Function &F, ArrayRef<Instruction *> SwitchTraceTargets); bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks, @@ -173,20 +161,21 @@ class ModuleSanitizerCoverageAFL void SetNoSanitizeMetadata(Instruction *I) { +#if LLVM_VERSION_MAJOR >= 16 + I->setMetadata(LLVMContext::MD_nosanitize, MDNode::get(*C, std::nullopt)); +#else I->setMetadata(I->getModule()->getMDKindID("nosanitize"), MDNode::get(*C, None)); +#endif } std::string getSectionName(const std::string &Section) const; std::string getSectionStart(const std::string &Section) const; std::string getSectionEnd(const std::string &Section) const; - FunctionCallee SanCovTracePCIndir; FunctionCallee SanCovTracePC, SanCovTracePCGuard; FunctionCallee SanCovTraceCmpFunction[4]; FunctionCallee SanCovTraceConstCmpFunction[4]; - FunctionCallee SanCovTraceDivFunction[2]; - FunctionCallee SanCovTraceGepFunction; FunctionCallee SanCovTraceSwitchFunction; GlobalVariable *SanCovLowestStack; Type *IntptrTy, *IntptrPtrTy, *Int64Ty, *Int64PtrTy, *Int32Ty, *Int32PtrTy, @@ -206,7 +195,7 @@ class ModuleSanitizerCoverageAFL SanitizerCoverageOptions Options; - uint32_t instr = 0, selects = 0, unhandled = 0; + uint32_t instr = 0, selects = 0, unhandled = 0, dump_cc = 0; GlobalVariable *AFLMapPtr = NULL; ConstantInt *One = NULL; ConstantInt *Zero = NULL; @@ -215,19 +204,21 @@ class ModuleSanitizerCoverageAFL } // namespace -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK llvmGetPassPluginInfo() { - return {LLVM_PLUGIN_API_VERSION, "SanitizerCoveragePCGUARD", "v0.1", + return {LLVM_PLUGIN_API_VERSION, "SanitizerCoveragePCGUARD", "v0.2", /* lambda to insert our pass into the pass pipeline. */ [](PassBuilder &PB) { - #if LLVM_VERSION_MAJOR <= 13 +#if LLVM_VERSION_MAJOR == 13 using OptimizationLevel = typename PassBuilder::OptimizationLevel; - #endif +#endif +#if LLVM_VERSION_MAJOR >= 16 + PB.registerOptimizerEarlyEPCallback( +#else PB.registerOptimizerLastEPCallback( +#endif [](ModulePassManager &MPM, OptimizationLevel OL) { MPM.addPass(ModuleSanitizerCoverageAFL()); @@ -238,8 +229,6 @@ llvmGetPassPluginInfo() { } -#endif - PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, ModuleAnalysisManager &MAM) { @@ -266,25 +255,18 @@ PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, std::pair<Value *, Value *> ModuleSanitizerCoverageAFL::CreateSecStartEnd( Module &M, const char *Section, Type *Ty) { - GlobalVariable *SecStart = - new GlobalVariable(M, -#if LLVM_VERSION_MAJOR >= 15 - Ty, -#else - Ty->getPointerElementType(), -#endif - false, GlobalVariable::ExternalWeakLinkage, nullptr, - getSectionStart(Section)); + // Use ExternalWeak so that if all sections are discarded due to section + // garbage collection, the linker will not report undefined symbol errors. + // Windows defines the start/stop symbols in compiler-rt so no need for + // ExternalWeak. + GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF() + ? GlobalVariable::ExternalLinkage + : GlobalVariable::ExternalWeakLinkage; + GlobalVariable *SecStart = new GlobalVariable(M, Ty, false, Linkage, nullptr, + getSectionStart(Section)); SecStart->setVisibility(GlobalValue::HiddenVisibility); - GlobalVariable *SecEnd = - new GlobalVariable(M, -#if LLVM_VERSION_MAJOR >= 15 - Ty, -#else - Ty->getPointerElementType(), -#endif - false, GlobalVariable::ExternalWeakLinkage, nullptr, - getSectionEnd(Section)); + GlobalVariable *SecEnd = new GlobalVariable(M, Ty, false, Linkage, nullptr, + getSectionEnd(Section)); SecEnd->setVisibility(GlobalValue::HiddenVisibility); IRBuilder<> IRB(M.getContext()); if (!TargetTriple.isOSBinFormatCOFF()) @@ -295,7 +277,8 @@ std::pair<Value *, Value *> ModuleSanitizerCoverageAFL::CreateSecStartEnd( auto SecStartI8Ptr = IRB.CreatePointerCast(SecStart, Int8PtrTy); auto GEP = IRB.CreateGEP(Int8Ty, SecStartI8Ptr, ConstantInt::get(IntptrTy, sizeof(uint64_t))); - return std::make_pair(IRB.CreatePointerCast(GEP, Ty), SecEnd); + return std::make_pair(IRB.CreatePointerCast(GEP, PointerType::getUnqual(Ty)), + SecEnd); } @@ -307,8 +290,9 @@ Function *ModuleSanitizerCoverageAFL::CreateInitCallsForSections( auto SecStart = SecStartEnd.first; auto SecEnd = SecStartEnd.second; Function *CtorFunc; + Type *PtrTy = PointerType::getUnqual(Ty); std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions( - M, CtorName, InitFunctionName, {Ty, Ty}, {SecStart, SecEnd}); + M, CtorName, InitFunctionName, {PtrTy, PtrTy}, {SecStart, SecEnd}); assert(CtorFunc->getName() == CtorName); if (TargetTriple.supportsCOMDAT()) { @@ -332,7 +316,6 @@ Function *ModuleSanitizerCoverageAFL::CreateInitCallsForSections( // to include the sancov constructor. This way the linker can deduplicate // the constructors but always leave one copy. CtorFunc->setLinkage(GlobalValue::WeakODRLinkage); - appendToUsed(M, CtorFunc); } @@ -344,37 +327,27 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( Module &M, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) { setvbuf(stdout, NULL, _IONBF, 0); - if (getenv("AFL_DEBUG")) debug = 1; + + if (getenv("AFL_DEBUG")) { debug = 1; } + + if (getenv("AFL_DUMP_CYCLOMATIC_COMPLEXITY")) { dump_cc = 1; } if ((isatty(2) && !getenv("AFL_QUIET")) || debug) { SAYF(cCYA "SanitizerCoveragePCGUARD" VERSION cRST "\n"); - } else + } else { be_quiet = 1; + } + skip_nozero = getenv("AFL_LLVM_SKIP_NEVERZERO"); use_threadsafe_counters = getenv("AFL_LLVM_THREADSAFE_INST"); initInstrumentList(); scanForDangerousFunctions(&M); - if (debug) { - - fprintf(stderr, - "SANCOV: covtype:%u indirect:%d stack:%d noprune:%d " - "createtable:%d tracepcguard:%d tracepc:%d\n", - Options.CoverageType, Options.IndirectCalls == true ? 1 : 0, - Options.StackDepth == true ? 1 : 0, Options.NoPrune == true ? 1 : 0, - // Options.InlineBoolFlag == true ? 1 : 0, - Options.PCTable == true ? 1 : 0, - Options.TracePCGuard == true ? 1 : 0, - Options.TracePC == true ? 1 : 0); - - } - - if (Options.CoverageType == SanitizerCoverageOptions::SCK_None) return false; C = &(M.getContext()); DL = &M.getDataLayout(); CurModule = &M; @@ -397,16 +370,14 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( Int16Ty = IRB.getInt16Ty(); Int8Ty = IRB.getInt8Ty(); Int1Ty = IRB.getInt1Ty(); - LLVMContext &Ctx = M.getContext(); + LLVMContext &Ctx = M.getContext(); AFLMapPtr = new GlobalVariable(M, PointerType::get(Int8Ty, 0), false, GlobalValue::ExternalLinkage, 0, "__afl_area_ptr"); One = ConstantInt::get(IntegerType::getInt8Ty(Ctx), 1); Zero = ConstantInt::get(IntegerType::getInt8Ty(Ctx), 0); - SanCovTracePCIndir = - M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy); // Make sure smaller parameters are zero-extended to i64 if required by the // target ABI. AttributeList SanCovTraceCmpZeroExtAL; @@ -436,26 +407,13 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( SanCovTraceConstCmpFunction[3] = M.getOrInsertFunction(SanCovTraceConstCmp8, VoidTy, Int64Ty, Int64Ty); - { - - AttributeList AL; - AL = AL.addParamAttribute(*C, 0, Attribute::ZExt); - SanCovTraceDivFunction[0] = - M.getOrInsertFunction(SanCovTraceDiv4, AL, VoidTy, IRB.getInt32Ty()); - - } - - SanCovTraceDivFunction[1] = - M.getOrInsertFunction(SanCovTraceDiv8, VoidTy, Int64Ty); - SanCovTraceGepFunction = - M.getOrInsertFunction(SanCovTraceGep, VoidTy, IntptrTy); SanCovTraceSwitchFunction = M.getOrInsertFunction(SanCovTraceSwitchName, VoidTy, Int64Ty, Int64PtrTy); Constant *SanCovLowestStackConstant = M.getOrInsertGlobal(SanCovLowestStackName, IntptrTy); SanCovLowestStack = dyn_cast<GlobalVariable>(SanCovLowestStackConstant); - if (!SanCovLowestStack) { + if (!SanCovLowestStack || SanCovLowestStack->getValueType() != IntptrTy) { C->emitError(StringRef("'") + SanCovLowestStackName + "' should not be declared by the user"); @@ -465,8 +423,6 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( SanCovLowestStack->setThreadLocalMode( GlobalValue::ThreadLocalMode::InitialExecTLSModel); - if (Options.StackDepth && !SanCovLowestStack->isDeclaration()) - SanCovLowestStack->setInitializer(Constant::getAllOnesValue(IntptrTy)); SanCovTracePC = M.getOrInsertFunction(SanCovTracePCName, VoidTy); SanCovTracePCGuard = @@ -481,40 +437,25 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( Ctor = CreateInitCallsForSections(M, SanCovModuleCtorTracePcGuardName, SanCovTracePCGuardInitName, Int32PtrTy, SanCovGuardsSectionName); - if (Function8bitCounterArray) - Ctor = CreateInitCallsForSections(M, SanCovModuleCtor8bitCountersName, - SanCov8bitCountersInitName, Int8PtrTy, - SanCovCountersSectionName); - if (FunctionBoolArray) { - Ctor = CreateInitCallsForSections(M, SanCovModuleCtorBoolFlagName, - SanCovBoolFlagInitName, Int1PtrTy, - SanCovBoolFlagSectionName); - - } + if (Ctor && debug) { - if (Ctor && Options.PCTable) { - - auto SecStartEnd = CreateSecStartEnd(M, SanCovPCsSectionName, IntptrPtrTy); - FunctionCallee InitFunction = declareSanitizerInitFunction( - M, SanCovPCsInitName, {IntptrPtrTy, IntptrPtrTy}); - IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator()); - IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second}); + fprintf(stderr, "SANCOV: installed pcguard_init in ctor\n"); } - // We don't reference these arrays directly in any of our runtime functions, - // so we need to prevent them from being dead stripped. - if (TargetTriple.isOSBinFormatMachO()) appendToUsed(M, GlobalsToAppendToUsed); + appendToUsed(M, GlobalsToAppendToUsed); appendToCompilerUsed(M, GlobalsToAppendToCompilerUsed); if (!be_quiet) { - if (!instr) + if (!instr) { + WARNF("No instrumentation targets found."); - else { - char modeline[100]; + } else { + + char modeline[128]; snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", @@ -535,39 +476,36 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( } // True if block has successors and it dominates all of them. -bool isFullDominator(const BasicBlock *BB, const DominatorTree *DT) { +static bool isFullDominator(const BasicBlock *BB, const DominatorTree *DT) { - if (succ_begin(BB) == succ_end(BB)) return false; + if (succ_empty(BB)) return false; - for (const BasicBlock *SUCC : make_range(succ_begin(BB), succ_end(BB))) { + return llvm::all_of(successors(BB), [&](const BasicBlock *SUCC) { - if (!DT->dominates(BB, SUCC)) return false; + return DT->dominates(BB, SUCC); - } - - return true; + }); } // True if block has predecessors and it postdominates all of them. -bool isFullPostDominator(const BasicBlock *BB, const PostDominatorTree *PDT) { +static bool isFullPostDominator(const BasicBlock *BB, + const PostDominatorTree *PDT) { - if (pred_begin(BB) == pred_end(BB)) return false; + if (pred_empty(BB)) return false; - for (const BasicBlock *PRED : make_range(pred_begin(BB), pred_end(BB))) { + return llvm::all_of(predecessors(BB), [&](const BasicBlock *PRED) { - if (!PDT->dominates(BB, PRED)) return false; + return PDT->dominates(BB, PRED); - } - - return true; + }); } -bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, - const DominatorTree *DT, - const PostDominatorTree *PDT, - const SanitizerCoverageOptions &Options) { +static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, + const DominatorTree *DT, + const PostDominatorTree *PDT, + const SanitizerCoverageOptions &Options) { // Don't insert coverage for blocks containing nothing but unreachable: we // will never call __sanitizer_cov() for them, so counting them in @@ -582,10 +520,6 @@ bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, if (Options.NoPrune || &F.getEntryBlock() == BB) return true; - if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function && - &F.getEntryBlock() != BB) - return false; - // Do not instrument full dominators, or full post-dominators with multiple // predecessors. return !isFullDominator(BB, DT) && @@ -597,41 +531,54 @@ bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, // A twist here is that we treat From->To as a backedge if // * To dominates From or // * To->UniqueSuccessor dominates From -bool IsBackEdge(BasicBlock *From, BasicBlock *To, const DominatorTree *DT) { +#if 0 +static bool IsBackEdge(BasicBlock *From, BasicBlock *To, + const DominatorTree *DT) { - if (DT->dominates(To, From)) return true; + if (DT->dominates(To, From)) + return true; if (auto Next = To->getUniqueSuccessor()) - if (DT->dominates(Next, From)) return true; + if (DT->dominates(Next, From)) + return true; return false; } +#endif + // Prunes uninteresting Cmp instrumentation: // * CMP instructions that feed into loop backedge branch. // // Note that Cmp pruning is controlled by the same flag as the // BB pruning. -bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree *DT, - const SanitizerCoverageOptions &Options) { +#if 0 +static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree *DT, + const SanitizerCoverageOptions &Options) { if (!Options.NoPrune) if (CMP->hasOneUse()) if (auto BR = dyn_cast<BranchInst>(CMP->user_back())) for (BasicBlock *B : BR->successors()) - if (IsBackEdge(BR->getParent(), B, DT)) return false; + if (IsBackEdge(BR->getParent(), B, DT)) + return false; return true; } +#endif + void ModuleSanitizerCoverageAFL::instrumentFunction( Function &F, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) { if (F.empty()) return; if (!isInInstrumentList(&F, FMNAME)) return; - if (F.getName().find(".module_ctor") != std::string::npos) return; // Should not instrument sanitizer init functions. +#if LLVM_VERSION_MAJOR >= 18 + if (F.getName().starts_with("__sanitizer_")) +#else if (F.getName().startswith("__sanitizer_")) +#endif return; // Don't instrument __sanitizer_* callbacks. // Don't touch available_externally functions, their actual body is elewhere. if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) return; @@ -647,15 +594,13 @@ void ModuleSanitizerCoverageAFL::instrumentFunction( if (F.hasPersonalityFn() && isAsynchronousEHPersonality(classifyEHPersonality(F.getPersonalityFn()))) return; + if (F.hasFnAttribute(Attribute::NoSanitizeCoverage)) return; if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) SplitAllCriticalEdges( F, CriticalEdgeSplittingOptions().setIgnoreUnreachableDests()); - SmallVector<Instruction *, 8> IndirCalls; - SmallVector<BasicBlock *, 16> BlocksToInstrument; - SmallVector<Instruction *, 8> CmpTraceTargets; - SmallVector<Instruction *, 8> SwitchTraceTargets; - SmallVector<BinaryOperator *, 8> DivTraceTargets; - SmallVector<GetElementPtrInst *, 8> GepTraceTargets; + SmallVector<BasicBlock *, 16> BlocksToInstrument; + SmallVector<Instruction *, 8> CmpTraceTargets; + SmallVector<Instruction *, 8> SwitchTraceTargets; const DominatorTree *DT = DTCallback(F); const PostDominatorTree *PDT = PDTCallback(F); @@ -665,47 +610,37 @@ void ModuleSanitizerCoverageAFL::instrumentFunction( if (shouldInstrumentBlock(F, &BB, DT, PDT, Options)) BlocksToInstrument.push_back(&BB); - for (auto &Inst : BB) { + /* + for (auto &Inst : BB) { - if (Options.IndirectCalls) { + if (Options.TraceCmp) { - CallBase *CB = dyn_cast<CallBase>(&Inst); - if (CB && !CB->getCalledFunction()) IndirCalls.push_back(&Inst); + if (ICmpInst *CMP = dyn_cast<ICmpInst>(&Inst)) + if (IsInterestingCmp(CMP, DT, Options)) + CmpTraceTargets.push_back(&Inst); + if (isa<SwitchInst>(&Inst)) + SwitchTraceTargets.push_back(&Inst); - } + } - if (Options.TraceCmp) { + } - if (ICmpInst *CMP = dyn_cast<ICmpInst>(&Inst)) - if (IsInterestingCmp(CMP, DT, Options)) - CmpTraceTargets.push_back(&Inst); - if (isa<SwitchInst>(&Inst)) SwitchTraceTargets.push_back(&Inst); + */ - } + } - if (Options.TraceDiv) - if (BinaryOperator *BO = dyn_cast<BinaryOperator>(&Inst)) - if (BO->getOpcode() == Instruction::SDiv || - BO->getOpcode() == Instruction::UDiv) - DivTraceTargets.push_back(BO); - if (Options.TraceGep) - if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&Inst)) - GepTraceTargets.push_back(GEP); - if (Options.StackDepth) - if (isa<InvokeInst>(Inst) || - (isa<CallInst>(Inst) && !isa<IntrinsicInst>(Inst))) - IsLeafFunc = false; + if (debug) { - } + fprintf(stderr, "SanitizerCoveragePCGUARD: instrumenting %s in %s\n", + F.getName().str().c_str(), F.getParent()->getName().str().c_str()); } InjectCoverage(F, BlocksToInstrument, IsLeafFunc); - InjectCoverageForIndirectCalls(F, IndirCalls); - InjectTraceForCmp(F, CmpTraceTargets); - InjectTraceForSwitch(F, SwitchTraceTargets); - InjectTraceForDiv(F, DivTraceTargets); - InjectTraceForGep(F, GepTraceTargets); + // InjectTraceForCmp(F, CmpTraceTargets); + // InjectTraceForSwitch(F, SwitchTraceTargets); + + if (dump_cc) { calcCyclomaticComplexity(&F); } } @@ -714,36 +649,33 @@ GlobalVariable *ModuleSanitizerCoverageAFL::CreateFunctionLocalArrayInSection( ArrayType *ArrayTy = ArrayType::get(Ty, NumElements); auto Array = new GlobalVariable( - *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, - Constant::getNullValue(ArrayTy), "__sancov_gen_"); + *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, + Constant::getNullValue(ArrayTy), "__sancov_gen_"); -#if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && (TargetTriple.isOSBinFormatELF() || !F.isInterposable())) if (auto Comdat = getOrCreateFunctionComdat(F, TargetTriple)) Array->setComdat(Comdat); -#else - if (TargetTriple.supportsCOMDAT() && !F.isInterposable()) - if (auto Comdat = - GetOrCreateFunctionComdat(F, TargetTriple, CurModuleUniqueId)) - Array->setComdat(Comdat); -#endif - Array->setSection(getSectionName(Section)); -#if (LLVM_VERSION_MAJOR >= 11) || \ - (LLVM_VERSION_MAJOR == 10 && LLVM_VERSION_MINOR >= 1) - #if LLVM_VERSION_MAJOR >= 16 +#if LLVM_VERSION_MAJOR >= 16 Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue())); - #else - Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedSize())); - #endif #else - Array->setAlignment(Align(4)); // cheating + Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedSize())); #endif - GlobalsToAppendToUsed.push_back(Array); - GlobalsToAppendToCompilerUsed.push_back(Array); - MDNode *MD = MDNode::get(F.getContext(), ValueAsMetadata::get(&F)); - Array->addMetadata(LLVMContext::MD_associated, *MD); + + // sancov_pcs parallels the other metadata section(s). Optimizers (e.g. + // GlobalOpt/ConstantMerge) may not discard sancov_pcs and the other + // section(s) as a unit, so we conservatively retain all unconditionally in + // the compiler. + // + // With comdat (COFF/ELF), the linker can guarantee the associated sections + // will be retained or discarded as a unit, so llvm.compiler.used is + // sufficient. Otherwise, conservatively make all of them retained by the + // linker. + if (Array->hasComdat()) + GlobalsToAppendToCompilerUsed.push_back(Array); + else + GlobalsToAppendToUsed.push_back(Array); return Array; @@ -768,8 +700,12 @@ GlobalVariable *ModuleSanitizerCoverageAFL::CreatePCArray( PCs.push_back((Constant *)IRB.CreatePointerCast( BlockAddress::get(AllBlocks[i]), IntptrPtrTy)); +#if LLVM_VERSION_MAJOR >= 16 + PCs.push_back(Constant::getNullValue(IntptrPtrTy)); +#else PCs.push_back((Constant *)IRB.CreateIntToPtr( ConstantInt::get(IntptrTy, 0), IntptrPtrTy)); +#endif } @@ -792,21 +728,13 @@ void ModuleSanitizerCoverageAFL::CreateFunctionLocalArrays( FunctionGuardArray = CreateFunctionLocalArrayInSection( AllBlocks.size() + special, F, Int32Ty, SanCovGuardsSectionName); - if (Options.Inline8bitCounters) - Function8bitCounterArray = CreateFunctionLocalArrayInSection( - AllBlocks.size(), F, Int8Ty, SanCovCountersSectionName); - /* - if (Options.InlineBoolFlag) - FunctionBoolArray = CreateFunctionLocalArrayInSection( - AllBlocks.size(), F, Int1Ty, SanCovBoolFlagSectionName); - */ - if (Options.PCTable) FunctionPCsArray = CreatePCArray(F, AllBlocks); - } bool ModuleSanitizerCoverageAFL::InjectCoverage( Function &F, ArrayRef<BasicBlock *> AllBlocks, bool IsLeafFunc) { + if (AllBlocks.empty()) return false; + uint32_t cnt_cov = 0, cnt_sel = 0, cnt_sel_inc = 0; static uint32_t first = 1; @@ -855,7 +783,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( } -#if (LLVM_VERSION_MAJOR >= 12) else if (t->getTypeID() == llvm::Type::FixedVectorTyID) { FixedVectorType *tt = dyn_cast<FixedVectorType>(t); @@ -868,16 +795,14 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( } -#endif - } } } - /* Create PCGUARD array */ CreateFunctionLocalArrays(F, AllBlocks, first + cnt_cov + cnt_sel_inc); + if (first) { first = 0; } selects += cnt_sel; @@ -889,12 +814,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( CallInst *callInst = nullptr; - /* - std::string errMsg; - raw_string_ostream os(errMsg); - IN.print(os); - fprintf(stderr, "X: %s\n", os.str().c_str()); - */ if ((callInst = dyn_cast<CallInst>(&IN))) { Function *Callee = callInst->getCalledFunction(); @@ -953,7 +872,7 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), ConstantInt::get( IntptrTy, - (cnt_cov + ++local_selects + AllBlocks.size()) * 4)), + (cnt_cov + local_selects++ + AllBlocks.size()) * 4)), Int32PtrTy); auto GuardPtr2 = IRB.CreateIntToPtr( @@ -961,7 +880,7 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), ConstantInt::get( IntptrTy, - (cnt_cov + ++local_selects + AllBlocks.size()) * 4)), + (cnt_cov + local_selects++ + AllBlocks.size()) * 4)), Int32PtrTy); result = IRB.CreateSelect(condition, GuardPtr1, GuardPtr2); @@ -998,7 +917,7 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), ConstantInt::get( IntptrTy, - (cnt_cov + ++local_selects + AllBlocks.size()) * 4)), + (cnt_cov + local_selects++ + AllBlocks.size()) * 4)), Int32PtrTy); x = IRB.CreateInsertElement(GuardPtr1, val1, (uint64_t)0); @@ -1007,7 +926,7 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), ConstantInt::get( IntptrTy, - (cnt_cov + ++local_selects + AllBlocks.size()) * 4)), + (cnt_cov + local_selects++ + AllBlocks.size()) * 4)), Int32PtrTy); y = IRB.CreateInsertElement(GuardPtr2, val2, (uint64_t)0); @@ -1016,7 +935,7 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( val1 = IRB.CreateIntToPtr( IRB.CreateAdd( IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), - ConstantInt::get(IntptrTy, (cnt_cov + ++local_selects + + ConstantInt::get(IntptrTy, (cnt_cov + local_selects++ + AllBlocks.size()) * 4)), Int32PtrTy); @@ -1025,7 +944,7 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( val2 = IRB.CreateIntToPtr( IRB.CreateAdd( IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), - ConstantInt::get(IntptrTy, (cnt_cov + ++local_selects + + ConstantInt::get(IntptrTy, (cnt_cov + local_selects++ + AllBlocks.size()) * 4)), Int32PtrTy); @@ -1033,12 +952,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( } - /* - std::string errMsg; - raw_string_ostream os(errMsg); - x->print(os); - fprintf(stderr, "X: %s\n", os.str().c_str()); - */ result = IRB.CreateSelect(condition, x, y); } @@ -1050,6 +963,7 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( #endif { + // fprintf(stderr, "UNHANDLED: %u\n", t->getTypeID()); unhandled++; continue; @@ -1063,13 +977,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( IRB.CreateLoad(PointerType::get(Int8Ty, 0), AFLMapPtr); ModuleSanitizerCoverageAFL::SetNoSanitizeMetadata(MapPtr); - /* - std::string errMsg; - raw_string_ostream os(errMsg); - result->print(os); - fprintf(stderr, "X: %s\n", os.str().c_str()); - */ - while (1) { /* Get CurLoc */ @@ -1159,29 +1066,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( } -// On every indirect call we call a run-time function -// __sanitizer_cov_indir_call* with two parameters: -// - callee address, -// - global cache array that contains CacheSize pointers (zero-initialized). -// The cache is used to speed up recording the caller-callee pairs. -// The address of the caller is passed implicitly via caller PC. -// CacheSize is encoded in the name of the run-time function. -void ModuleSanitizerCoverageAFL::InjectCoverageForIndirectCalls( - Function &F, ArrayRef<Instruction *> IndirCalls) { - - if (IndirCalls.empty()) return; - for (auto I : IndirCalls) { - - IRBuilder<> IRB(I); - CallBase &CB = cast<CallBase>(*I); - Value *Callee = CB.getCalledOperand(); - if (isa<InlineAsm>(Callee)) continue; - IRB.CreateCall(SanCovTracePCIndir, IRB.CreatePointerCast(Callee, IntptrTy)); - - } - -} - // For every switch statement we insert a call: // __sanitizer_cov_trace_switch(CondValue, // {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... }) @@ -1237,41 +1121,6 @@ void ModuleSanitizerCoverageAFL::InjectTraceForSwitch( } -void ModuleSanitizerCoverageAFL::InjectTraceForDiv( - Function &, ArrayRef<BinaryOperator *> DivTraceTargets) { - - for (auto BO : DivTraceTargets) { - - IRBuilder<> IRB(BO); - Value *A1 = BO->getOperand(1); - if (isa<ConstantInt>(A1)) continue; - if (!A1->getType()->isIntegerTy()) continue; - uint64_t TypeSize = DL->getTypeStoreSizeInBits(A1->getType()); - int CallbackIdx = TypeSize == 32 ? 0 : TypeSize == 64 ? 1 : -1; - if (CallbackIdx < 0) continue; - auto Ty = Type::getIntNTy(*C, TypeSize); - IRB.CreateCall(SanCovTraceDivFunction[CallbackIdx], - {IRB.CreateIntCast(A1, Ty, true)}); - - } - -} - -void ModuleSanitizerCoverageAFL::InjectTraceForGep( - Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) { - - for (auto GEP : GepTraceTargets) { - - IRBuilder<> IRB(GEP); - for (Use &Idx : GEP->indices()) - if (!isa<ConstantInt>(Idx) && Idx->getType()->isIntegerTy()) - IRB.CreateCall(SanCovTraceGepFunction, - {IRB.CreateIntCast(Idx, IntptrTy, true)}); - - } - -} - void ModuleSanitizerCoverageAFL::InjectTraceForCmp( Function &, ArrayRef<Instruction *> CmpTraceTargets) { @@ -1321,27 +1170,44 @@ void ModuleSanitizerCoverageAFL::InjectCoverageAtBlock(Function &F, BasicBlock::iterator IP = BB.getFirstInsertionPt(); bool IsEntryBB = &BB == &F.getEntryBlock(); + DebugLoc EntryLoc; if (IsEntryBB) { - // Keep allocas and llvm.localescape calls in the entry block. Even + if (auto SP = F.getSubprogram()) + EntryLoc = DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP); + // Keep static allocas and llvm.localescape calls in the entry block. Even // if we aren't splitting the block, it's nice for allocas to be before // calls. IP = PrepareToSplitEntryBlock(BB, IP); +#if LLVM_VERSION_MAJOR < 15 - } - - IRBuilder<> IRB(&*IP); - - if (Options.TracePC) { + } else { - IRB.CreateCall(SanCovTracePC); - // ->setCannotMerge(); // gets the PC using GET_CALLER_PC. + EntryLoc = IP->getDebugLoc(); + if (!EntryLoc) + if (auto *SP = F.getSubprogram()) + EntryLoc = DILocation::get(SP->getContext(), 0, 0, SP); +#endif } +#if LLVM_VERSION_MAJOR >= 16 + InstrumentationIRBuilder IRB(&*IP); +#else + IRBuilder<> IRB(&*IP); +#endif + if (EntryLoc) IRB.SetCurrentDebugLocation(EntryLoc); if (Options.TracePCGuard) { + /* + auto GuardPtr = IRB.CreateIntToPtr( + IRB.CreateAdd(IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), + ConstantInt::get(IntptrTy, Idx * 4)), + Int32PtrTy); + IRB.CreateCall(SanCovTracePCGuard, GuardPtr)->setCannotMerge(); + */ + /* Get CurLoc */ Value *GuardPtr = IRB.CreateIntToPtr( @@ -1399,57 +1265,6 @@ void ModuleSanitizerCoverageAFL::InjectCoverageAtBlock(Function &F, } - if (Options.Inline8bitCounters) { - - auto CounterPtr = IRB.CreateGEP( - Function8bitCounterArray->getValueType(), Function8bitCounterArray, - {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)}); - auto Load = IRB.CreateLoad(Int8Ty, CounterPtr); - auto Inc = IRB.CreateAdd(Load, ConstantInt::get(Int8Ty, 1)); - auto Store = IRB.CreateStore(Inc, CounterPtr); - SetNoSanitizeMetadata(Load); - SetNoSanitizeMetadata(Store); - - } - - /* - if (Options.InlineBoolFlag) { - - auto FlagPtr = IRB.CreateGEP( - FunctionBoolArray->getValueType(), FunctionBoolArray, - {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)}); - auto Load = IRB.CreateLoad(Int1Ty, FlagPtr); - auto ThenTerm = - SplitBlockAndInsertIfThen(IRB.CreateIsNull(Load), &*IP, false); - IRBuilder<> ThenIRB(ThenTerm); - auto Store = ThenIRB.CreateStore(ConstantInt::getTrue(Int1Ty), FlagPtr); - SetNoSanitizeMetadata(Load); - SetNoSanitizeMetadata(Store); - - } - - */ - - if (Options.StackDepth && IsEntryBB && !IsLeafFunc) { - - // Check stack depth. If it's the deepest so far, record it. - Module *M = F.getParent(); - Function *GetFrameAddr = Intrinsic::getDeclaration( - M, Intrinsic::frameaddress, - IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace())); - auto FrameAddrPtr = - IRB.CreateCall(GetFrameAddr, {Constant::getNullValue(Int32Ty)}); - auto FrameAddrInt = IRB.CreatePtrToInt(FrameAddrPtr, IntptrTy); - auto LowestStack = IRB.CreateLoad(IntptrTy, SanCovLowestStack); - auto IsStackLower = IRB.CreateICmpULT(FrameAddrInt, LowestStack); - auto ThenTerm = SplitBlockAndInsertIfThen(IsStackLower, &*IP, false); - IRBuilder<> ThenIRB(ThenTerm); - auto Store = ThenIRB.CreateStore(FrameAddrInt, SanCovLowestStack); - SetNoSanitizeMetadata(LowestStack); - SetNoSanitizeMetadata(Store); - - } - } std::string ModuleSanitizerCoverageAFL::getSectionName( diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 5372fae0..e450dc45 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -3,7 +3,7 @@ ------------------------------------------------ Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,6 +22,10 @@ #define __USE_GNU #endif #include <dlfcn.h> + +__attribute__((weak)) void __sanitizer_symbolize_pc(void *, const char *fmt, + char *out_buf, + size_t out_buf_size); #endif #ifdef __ANDROID__ @@ -48,7 +52,7 @@ #include <errno.h> #include <sys/mman.h> -#ifndef __HAIKU__ +#if !defined(__HAIKU__) && !defined(__OpenBSD__) #include <sys/syscall.h> #endif #ifndef USEMMAP @@ -83,21 +87,21 @@ #include <sys/mman.h> #include <fcntl.h> +#ifdef AFL_PERSISTENT_RECORD + #include "afl-persistent-replay.h" +#endif + /* Globals needed by the injected instrumentation. The __afl_area_initial region is used for instrumentation output before __afl_map_shm() has a chance to run. It will end up as .comm, so it shouldn't be too wasteful. */ -#if MAP_SIZE <= 65536 - #define MAP_INITIAL_SIZE 2097152 -#else - #define MAP_INITIAL_SIZE MAP_SIZE -#endif - #if defined(__HAIKU__) extern ssize_t _kern_write(int fd, off_t pos, const void *buffer, size_t bufferSize); #endif // HAIKU +char *strcasestr(const char *haystack, const char *needle); + static u8 __afl_area_initial[MAP_INITIAL_SIZE]; static u8 *__afl_area_ptr_dummy = __afl_area_initial; static u8 *__afl_area_ptr_backup = __afl_area_initial; @@ -128,8 +132,8 @@ struct afl_module_info_t { uintptr_t base_address; // PC Guard start/stop - u32 start; - u32 stop; + u32 *start; + u32 *stop; // PC Table begin/end const uintptr_t *pcs_beg; @@ -151,6 +155,18 @@ afl_module_info_t *__afl_module_info = NULL; u32 __afl_pcmap_size = 0; uintptr_t *__afl_pcmap_ptr = NULL; + +typedef struct { + + uintptr_t start; + u32 len; + +} FilterPCEntry; + +u32 __afl_filter_pcs_size = 0; +FilterPCEntry *__afl_filter_pcs = NULL; +u8 *__afl_filter_pcs_module = NULL; + #endif // __AFL_CODE_COVERAGE /* 1 if we are running in afl, and the forkserver was started, else 0 */ @@ -174,6 +190,8 @@ __thread u32 __afl_prev_ctx; struct cmp_map *__afl_cmp_map; struct cmp_map *__afl_cmp_map_backup; +static u8 __afl_cmplog_max_len = 32; // 16-32 + /* Child pid? */ static s32 child_pid; @@ -189,7 +207,7 @@ static u8 _is_sancov; /* Debug? */ -static u32 __afl_debug; +/*static*/ u32 __afl_debug; /* Already initialized markers */ @@ -252,7 +270,7 @@ static void send_forkserver_error(int error) { u32 status; if (!error || error > 0xffff) return; - status = (FS_OPT_ERROR | FS_OPT_SET_ERROR(error)); + status = (FS_NEW_ERROR | error); if (write(FORKSRV_FD + 1, (char *)&status, 4) != 4) { return; } } @@ -355,32 +373,13 @@ static void __afl_map_shm(void) { if ((ptr = getenv("AFL_MAP_SIZE")) != NULL) { val = atoi(ptr); } if (val < __afl_final_loc) { - if (__afl_final_loc > FS_OPT_MAX_MAPSIZE) { - - if (!getenv("AFL_QUIET")) - fprintf(stderr, - "Error: AFL++ tools *require* to set AFL_MAP_SIZE to %u " - "to be able to run this instrumented program!\n", - __afl_final_loc); - - if (id_str) { - - send_forkserver_error(FS_ERROR_MAP_SIZE); - exit(-1); - - } + if (__afl_final_loc > MAP_INITIAL_SIZE && !getenv("AFL_QUIET")) { - } else { - - if (__afl_final_loc > MAP_INITIAL_SIZE && !getenv("AFL_QUIET")) { - - fprintf(stderr, - "Warning: AFL++ tools might need to set AFL_MAP_SIZE to %u " - "to be able to run this instrumented program if this " - "crashes!\n", - __afl_final_loc); - - } + fprintf(stderr, + "Warning: AFL++ tools might need to set AFL_MAP_SIZE to %u " + "to be able to run this instrumented program if this " + "crashes!\n", + __afl_final_loc); } @@ -388,15 +387,6 @@ static void __afl_map_shm(void) { } - } else { - - if (getenv("AFL_DUMP_MAP_SIZE")) { - - printf("%u\n", MAP_SIZE); - exit(-1); - - } - } if (__afl_sharedmem_fuzzing && (!id_str || !getenv(SHM_FUZZ_ENV_VAR) || @@ -462,14 +452,13 @@ static void __afl_map_shm(void) { if (__afl_debug) { - fprintf( - stderr, - "DEBUG: (1) id_str %s, __afl_area_ptr %p, __afl_area_initial %p, " - "__afl_area_ptr_dummy %p, __afl_map_addr 0x%llx, MAP_SIZE %u, " - "__afl_final_loc %u, __afl_map_size %u, max_size_forkserver %u/0x%x\n", - id_str == NULL ? "<null>" : id_str, __afl_area_ptr, __afl_area_initial, - __afl_area_ptr_dummy, __afl_map_addr, MAP_SIZE, __afl_final_loc, - __afl_map_size, FS_OPT_MAX_MAPSIZE, FS_OPT_MAX_MAPSIZE); + fprintf(stderr, + "DEBUG: (1) id_str %s, __afl_area_ptr %p, __afl_area_initial %p, " + "__afl_area_ptr_dummy %p, __afl_map_addr 0x%llx, MAP_SIZE %u, " + "__afl_final_loc %u, __afl_map_size %u\n", + id_str == NULL ? "<null>" : id_str, __afl_area_ptr, + __afl_area_initial, __afl_area_ptr_dummy, __afl_map_addr, MAP_SIZE, + __afl_final_loc, __afl_map_size); } @@ -544,12 +533,12 @@ static void __afl_map_shm(void) { if (__afl_map_size && __afl_map_size > MAP_SIZE) { - u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); - if (!map_env || atoi((char *)map_env) < MAP_SIZE) { + u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); + if (!map_env || atoi((char *)map_env) < MAP_SIZE) { - fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); - send_forkserver_error(FS_ERROR_MAP_SIZE); - _exit(1); + fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); + send_forkserver_error(FS_ERROR_MAP_SIZE); + _exit(1); } @@ -561,13 +550,13 @@ static void __afl_map_shm(void) { if (!__afl_area_ptr || __afl_area_ptr == (void *)-1) { - if (__afl_map_addr) + if (__afl_map_addr) send_forkserver_error(FS_ERROR_MAP_ADDR); else send_forkserver_error(FS_ERROR_SHMAT); perror("shmat for map"); - _exit(1); + _exit(1); } @@ -627,12 +616,10 @@ static void __afl_map_shm(void) { fprintf(stderr, "DEBUG: (2) id_str %s, __afl_area_ptr %p, __afl_area_initial %p, " "__afl_area_ptr_dummy %p, __afl_map_addr 0x%llx, MAP_SIZE " - "%u, __afl_final_loc %u, __afl_map_size %u, " - "max_size_forkserver %u/0x%x\n", + "%u, __afl_final_loc %u, __afl_map_size %u", id_str == NULL ? "<null>" : id_str, __afl_area_ptr, __afl_area_initial, __afl_area_ptr_dummy, __afl_map_addr, MAP_SIZE, - __afl_final_loc, __afl_map_size, FS_OPT_MAX_MAPSIZE, - FS_OPT_MAX_MAPSIZE); + __afl_final_loc, __afl_map_size); } @@ -673,7 +660,8 @@ static void __afl_map_shm(void) { if (id_str) { - if ((__afl_dummy_fd[1] = open("/dev/null", O_WRONLY)) < 0) { + // /dev/null doesn't work so we use /dev/urandom + if ((__afl_dummy_fd[1] = open("/dev/urandom", O_WRONLY)) < 0) { if (pipe(__afl_dummy_fd) < 0) { __afl_dummy_fd[1] = 1; } @@ -748,6 +736,19 @@ static void __afl_map_shm(void) { #endif // __AFL_CODE_COVERAGE + if (!__afl_cmp_map && getenv("AFL_CMPLOG_DEBUG")) { + + __afl_cmp_map_backup = __afl_cmp_map = malloc(sizeof(struct cmp_map)); + + } + + if (getenv("AFL_CMPLOG_MAX_LEN")) { + + int tmp = atoi(getenv("AFL_CMPLOG_MAX_LEN")); + if (tmp >= 16 && tmp <= 32) { __afl_cmplog_max_len = tmp; } + + } + } /* unmap SHM. */ @@ -842,242 +843,6 @@ void write_error_with_location(char *text, char *filename, int linenumber) { } -#ifdef __linux__ -static void __afl_start_snapshots(void) { - - static u8 tmp[4] = {0, 0, 0, 0}; - u32 status = 0; - u32 already_read_first = 0; - u32 was_killed; - - u8 child_stopped = 0; - - void (*old_sigchld_handler)(int) = signal(SIGCHLD, SIG_DFL); - - /* Phone home and tell the parent that we're OK. If parent isn't there, - assume we're not running in forkserver mode and just execute program. */ - - status |= (FS_OPT_ENABLED | FS_OPT_SNAPSHOT | FS_OPT_NEWCMPLOG); - if (__afl_sharedmem_fuzzing) { status |= FS_OPT_SHDMEM_FUZZ; } - if (__afl_map_size <= FS_OPT_MAX_MAPSIZE) - status |= (FS_OPT_SET_MAPSIZE(__afl_map_size) | FS_OPT_MAPSIZE); - if (__afl_dictionary_len && __afl_dictionary) { status |= FS_OPT_AUTODICT; } - memcpy(tmp, &status, 4); - - if (write(FORKSRV_FD + 1, tmp, 4) != 4) { return; } - - if (__afl_sharedmem_fuzzing || (__afl_dictionary_len && __afl_dictionary)) { - - if (read(FORKSRV_FD, &was_killed, 4) != 4) { - - write_error("read to afl-fuzz"); - _exit(1); - - } - - if (__afl_debug) { - - fprintf(stderr, "target forkserver recv: %08x\n", was_killed); - - } - - if ((was_killed & (FS_OPT_ENABLED | FS_OPT_SHDMEM_FUZZ)) == - (FS_OPT_ENABLED | FS_OPT_SHDMEM_FUZZ)) { - - __afl_map_shm_fuzz(); - - } - - if ((was_killed & (FS_OPT_ENABLED | FS_OPT_AUTODICT)) == - (FS_OPT_ENABLED | FS_OPT_AUTODICT) && - __afl_dictionary_len && __afl_dictionary) { - - // great lets pass the dictionary through the forkserver FD - u32 len = __afl_dictionary_len, offset = 0; - s32 ret; - - if (write(FORKSRV_FD + 1, &len, 4) != 4) { - - write(2, "Error: could not send dictionary len\n", - strlen("Error: could not send dictionary len\n")); - _exit(1); - - } - - while (len != 0) { - - ret = write(FORKSRV_FD + 1, __afl_dictionary + offset, len); - - if (ret < 1) { - - write(2, "Error: could not send dictionary\n", - strlen("Error: could not send dictionary\n")); - _exit(1); - - } - - len -= ret; - offset += ret; - - } - - } else { - - // uh this forkserver does not understand extended option passing - // or does not want the dictionary - if (!__afl_fuzz_ptr) already_read_first = 1; - - } - - } - - while (1) { - - int status; - - if (already_read_first) { - - already_read_first = 0; - - } else { - - /* Wait for parent by reading from the pipe. Abort if read fails. */ - if (read(FORKSRV_FD, &was_killed, 4) != 4) { - - write_error("reading from afl-fuzz"); - _exit(1); - - } - - } - - #ifdef _AFL_DOCUMENT_MUTATIONS - if (__afl_fuzz_ptr) { - - static uint32_t counter = 0; - char fn[32]; - sprintf(fn, "%09u:forkserver", counter); - s32 fd_doc = open(fn, O_WRONLY | O_CREAT | O_TRUNC, DEFAULT_PERMISSION); - if (fd_doc >= 0) { - - if (write(fd_doc, __afl_fuzz_ptr, *__afl_fuzz_len) != *__afl_fuzz_len) { - - fprintf(stderr, "write of mutation file failed: %s\n", fn); - unlink(fn); - - } - - close(fd_doc); - - } - - counter++; - - } - - #endif - - /* If we stopped the child in persistent mode, but there was a race - condition and afl-fuzz already issued SIGKILL, write off the old - process. */ - - if (child_stopped && was_killed) { - - child_stopped = 0; - if (waitpid(child_pid, &status, 0) < 0) { - - write_error("child_stopped && was_killed"); - _exit(1); // TODO why exit? - - } - - } - - if (!child_stopped) { - - /* Once woken up, create a clone of our process. */ - - child_pid = fork(); - if (child_pid < 0) { - - write_error("fork"); - _exit(1); - - } - - /* In child process: close fds, resume execution. */ - - if (!child_pid) { - - //(void)nice(-20); // does not seem to improve - - signal(SIGCHLD, old_sigchld_handler); - signal(SIGTERM, old_sigterm_handler); - - close(FORKSRV_FD); - close(FORKSRV_FD + 1); - - if (!afl_snapshot_take(AFL_SNAPSHOT_MMAP | AFL_SNAPSHOT_FDS | - AFL_SNAPSHOT_REGS | AFL_SNAPSHOT_EXIT)) { - - raise(SIGSTOP); - - } - - __afl_area_ptr[0] = 1; - memset(__afl_prev_loc, 0, NGRAM_SIZE_MAX * sizeof(PREV_LOC_T)); - - return; - - } - - } else { - - /* Special handling for persistent mode: if the child is alive but - currently stopped, simply restart it with SIGCONT. */ - - kill(child_pid, SIGCONT); - child_stopped = 0; - - } - - /* In parent process: write PID to pipe, then wait for child. */ - - if (write(FORKSRV_FD + 1, &child_pid, 4) != 4) { - - write_error("write to afl-fuzz"); - _exit(1); - - } - - if (waitpid(child_pid, &status, WUNTRACED) < 0) { - - write_error("waitpid"); - _exit(1); - - } - - /* In persistent mode, the child stops itself with SIGSTOP to indicate - a successful run. In this case, we want to wake it up without forking - again. */ - - if (WIFSTOPPED(status)) child_stopped = 1; - - /* Relay wait status to pipe, then loop back. */ - - if (write(FORKSRV_FD + 1, &status, 4) != 4) { - - write_error("writing to afl-fuzz"); - _exit(1); - - } - - } - -} - -#endif - /* Fork server logic. */ static void __afl_start_forkserver(void) { @@ -1090,113 +855,92 @@ static void __afl_start_forkserver(void) { old_sigterm_handler = orig_action.sa_handler; signal(SIGTERM, at_exit); -#ifdef __linux__ - if (/*!is_persistent &&*/ !__afl_cmp_map && !getenv("AFL_NO_SNAPSHOT") && - afl_snapshot_init() >= 0) { - - __afl_start_snapshots(); - return; - - } - -#endif - - u8 tmp[4] = {0, 0, 0, 0}; - u32 status_for_fsrv = 0; u32 already_read_first = 0; u32 was_killed; + u32 version = 0x41464c00 + FS_NEW_VERSION_MAX; + u32 tmp = version ^ 0xffffffff, status2, status = version; + u8 *msg = (u8 *)&status; + u8 *reply = (u8 *)&status2; u8 child_stopped = 0; void (*old_sigchld_handler)(int) = signal(SIGCHLD, SIG_DFL); - if (__afl_map_size <= FS_OPT_MAX_MAPSIZE) { + /* Phone home and tell the parent that we're OK. If parent isn't there, + assume we're not running in forkserver mode and just execute program. */ - status_for_fsrv |= (FS_OPT_SET_MAPSIZE(__afl_map_size) | FS_OPT_MAPSIZE); + // return because possible non-forkserver usage + if (write(FORKSRV_FD + 1, msg, 4) != 4) { return; } - } + if (read(FORKSRV_FD, reply, 4) != 4) { _exit(1); } + if (tmp != status2) { - if (__afl_dictionary_len && __afl_dictionary) { - - status_for_fsrv |= FS_OPT_AUTODICT; + write_error("wrong forkserver message from AFL++ tool"); + _exit(1); } - if (__afl_sharedmem_fuzzing) { status_for_fsrv |= FS_OPT_SHDMEM_FUZZ; } - if (status_for_fsrv) { + // send the set/requested options to forkserver + status = FS_NEW_OPT_MAPSIZE; // we always send the map size + if (__afl_sharedmem_fuzzing) { status |= FS_NEW_OPT_SHDMEM_FUZZ; } + if (__afl_dictionary_len && __afl_dictionary) { - status_for_fsrv |= (FS_OPT_ENABLED | FS_OPT_NEWCMPLOG); + status |= FS_NEW_OPT_AUTODICT; } - memcpy(tmp, &status_for_fsrv, 4); - - /* Phone home and tell the parent that we're OK. If parent isn't there, - assume we're not running in forkserver mode and just execute program. */ - - if (write(FORKSRV_FD + 1, tmp, 4) != 4) { return; } - - __afl_connected = 1; + if (write(FORKSRV_FD + 1, msg, 4) != 4) { _exit(1); } - if (__afl_sharedmem_fuzzing || (__afl_dictionary_len && __afl_dictionary)) { + // Now send the parameters for the set options, increasing by option number - if (read(FORKSRV_FD, &was_killed, 4) != 4) _exit(1); + // FS_NEW_OPT_MAPSIZE - we always send the map size + status = __afl_map_size; + if (write(FORKSRV_FD + 1, msg, 4) != 4) { _exit(1); } - if (__afl_debug) { + // FS_NEW_OPT_SHDMEM_FUZZ - no data - fprintf(stderr, "target forkserver recv: %08x\n", was_killed); + // FS_NEW_OPT_AUTODICT - send autodictionary + if (__afl_dictionary_len && __afl_dictionary) { - } + // pass the dictionary through the forkserver FD + u32 len = __afl_dictionary_len, offset = 0; - if ((was_killed & (FS_OPT_ENABLED | FS_OPT_SHDMEM_FUZZ)) == - (FS_OPT_ENABLED | FS_OPT_SHDMEM_FUZZ)) { + if (write(FORKSRV_FD + 1, &len, 4) != 4) { - __afl_map_shm_fuzz(); + write(2, "Error: could not send dictionary len\n", + strlen("Error: could not send dictionary len\n")); + _exit(1); } - if ((was_killed & (FS_OPT_ENABLED | FS_OPT_AUTODICT)) == - (FS_OPT_ENABLED | FS_OPT_AUTODICT) && - __afl_dictionary_len && __afl_dictionary) { + while (len != 0) { - // great lets pass the dictionary through the forkserver FD - u32 len = __afl_dictionary_len, offset = 0; + s32 ret; + ret = write(FORKSRV_FD + 1, __afl_dictionary + offset, len); - if (write(FORKSRV_FD + 1, &len, 4) != 4) { + if (ret < 1) { - write(2, "Error: could not send dictionary len\n", - strlen("Error: could not send dictionary len\n")); + write_error("could not send dictionary"); _exit(1); } - while (len != 0) { - - s32 ret; - ret = write(FORKSRV_FD + 1, __afl_dictionary + offset, len); - - if (ret < 1) { + len -= ret; + offset += ret; - write(2, "Error: could not send dictionary\n", - strlen("Error: could not send dictionary\n")); - _exit(1); - - } - - len -= ret; - offset += ret; + } - } + } - } else { + // send welcome message as final message + status = version; + if (write(FORKSRV_FD + 1, msg, 4) != 4) { _exit(1); } - // uh this forkserver does not understand extended option passing - // or does not want the dictionary - if (!__afl_fuzz_ptr) already_read_first = 1; + // END forkserver handshake - } + __afl_connected = 1; - } + if (__afl_sharedmem_fuzzing) { __afl_map_shm_fuzz(); } while (1) { @@ -1212,7 +956,7 @@ static void __afl_start_forkserver(void) { if (read(FORKSRV_FD, &was_killed, 4) != 4) { - // write_error("read from afl-fuzz"); + write_error("read from AFL++ tool"); _exit(1); } @@ -1341,6 +1085,10 @@ int __afl_persistent_loop(unsigned int max_cnt) { static u8 first_pass = 1; static u32 cycle_cnt; +#ifdef AFL_PERSISTENT_RECORD + char tcase[PATH_MAX]; +#endif + if (first_pass) { /* Make sure that every iteration of __AFL_LOOP() starts with a clean slate. @@ -1352,14 +1100,59 @@ int __afl_persistent_loop(unsigned int max_cnt) { __afl_area_ptr[0] = 1; memset(__afl_prev_loc, 0, NGRAM_SIZE_MAX * sizeof(PREV_LOC_T)); - cycle_cnt = max_cnt; first_pass = 0; __afl_selective_coverage_temp = 1; +#ifdef AFL_PERSISTENT_RECORD + if (unlikely(is_replay_record)) { + + cycle_cnt = replay_record_cnt; + goto persistent_record; + + } else + +#endif + { + + cycle_cnt = max_cnt; + + } + return 1; } else if (--cycle_cnt) { +#ifdef AFL_PERSISTENT_RECORD + if (unlikely(is_replay_record)) { + + persistent_record: + + snprintf(tcase, PATH_MAX, "%s/%s", + replay_record_dir ? replay_record_dir : "./", + record_list[replay_record_cnt - cycle_cnt]->d_name); + + #ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + if (unlikely(record_arg)) { + + *record_arg = tcase; + + } else + + #endif // AFL_PERSISTENT_REPLAY_ARGPARSE + { + + int fd = open(tcase, O_RDONLY); + dup2(fd, 0); + close(fd); + + } + + return 1; + + } + +#endif + raise(SIGSTOP); __afl_area_ptr[0] = 1; @@ -1477,6 +1270,7 @@ __attribute__((constructor(1))) void __afl_auto_second(void) { __afl_debug = 1; fprintf(stderr, "DEBUG: debug enabled\n"); + fprintf(stderr, "DEBUG: AFL++ afl-compiler-rt" VERSION "\n"); } @@ -1589,15 +1383,116 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *guard) { } #ifdef __AFL_CODE_COVERAGE -void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, - const uintptr_t *pcs_end) { +void afl_read_pc_filter_file(const char *filter_file) { - if (__afl_debug) { + FILE *file; + char ch; + + file = fopen(filter_file, "r"); + if (file == NULL) { + + perror("Error opening file"); + return; + + } + + // Check how many PCs we expect to read + while ((ch = fgetc(file)) != EOF) { + + if (ch == '\n') { __afl_filter_pcs_size++; } + + } + + // Rewind to actually read the PCs + fseek(file, 0, SEEK_SET); + + __afl_filter_pcs = malloc(__afl_filter_pcs_size * sizeof(FilterPCEntry)); + if (!__afl_filter_pcs) { + + perror("Error allocating PC array"); + return; + + } + + for (size_t i = 0; i < __afl_filter_pcs_size; i++) { + + fscanf(file, "%lx", &(__afl_filter_pcs[i].start)); + ch = fgetc(file); // Read tab + fscanf(file, "%u", &(__afl_filter_pcs[i].len)); + ch = fgetc(file); // Read tab + + if (!__afl_filter_pcs_module) { + + // Read the module name and store it. + // TODO: We only support one module here right now although + // there is technically no reason to support multiple modules + // in one go. + size_t max_module_len = 255; + size_t i = 0; + __afl_filter_pcs_module = malloc(max_module_len); + while (i < max_module_len - 1 && + (__afl_filter_pcs_module[i] = fgetc(file)) != '\t') { + + ++i; + + } - fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init called\n"); + __afl_filter_pcs_module[i] = '\0'; + fprintf(stderr, "DEBUGXXX: Read module name %s\n", + __afl_filter_pcs_module); + + } + + while ((ch = fgetc(file)) != '\n' && ch != EOF) + ; } + fclose(file); + +} + +u32 locate_in_pcs(uintptr_t needle, u32 *index) { + + size_t lower_bound = 0; + size_t upper_bound = __afl_filter_pcs_size - 1; + + while (lower_bound < __afl_filter_pcs_size && lower_bound <= upper_bound) { + + size_t current_index = lower_bound + (upper_bound - lower_bound) / 2; + + if (__afl_filter_pcs[current_index].start <= needle) { + + if (__afl_filter_pcs[current_index].start + + __afl_filter_pcs[current_index].len > + needle) { + + // Hit + *index = current_index; + return 1; + + } else { + + lower_bound = current_index + 1; + + } + + } else { + + if (!current_index) { break; } + upper_bound = current_index - 1; + + } + + } + + return 0; + +} + +void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + const uintptr_t *pcs_end) { + // If for whatever reason, we cannot get dlinfo here, then pc_guard_init also // couldn't get it and we'd end up attributing to the wrong module. Dl_info dlinfo; @@ -1610,6 +1505,16 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, } + if (__afl_debug) { + + fprintf( + stderr, + "DEBUG: (%u) __sanitizer_cov_pcs_init called for module %s with %ld " + "PCs\n", + getpid(), dlinfo.dli_fname, pcs_end - pcs_beg); + + } + afl_module_info_t *last_module_info = __afl_module_info; while (last_module_info && last_module_info->next) { @@ -1625,34 +1530,78 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, } + if (strcmp(dlinfo.dli_fname, last_module_info->name)) { + + // This can happen with modules being loaded after the forkserver + // where we decide to not track the module. In that case we must + // not track it here either. + fprintf( + stderr, + "WARNING: __sanitizer_cov_pcs_init module info mismatch: %s vs %s\n", + dlinfo.dli_fname, last_module_info->name); + return; + + } + last_module_info->pcs_beg = pcs_beg; last_module_info->pcs_end = pcs_end; + // This is a direct filter based on symbolizing inside the runtime. + // It should only be used with smaller binaries to avoid long startup + // times. Currently, this only supports a single token to scan for. + const char *pc_filter = getenv("AFL_PC_FILTER"); + + // This is a much faster PC filter based on pre-symbolized input data + // that is sorted for fast lookup through binary search. This method + // of filtering is suitable even for very large binaries. + const char *pc_filter_file = getenv("AFL_PC_FILTER_FILE"); + if (pc_filter_file && !__afl_filter_pcs) { + + afl_read_pc_filter_file(pc_filter_file); + + } + // Now update the pcmap. If this is the last module coming in, after all // pre-loaded code, then this will also map all of our delayed previous // modules. - - if (!__afl_pcmap_ptr) { return; } - + // for (afl_module_info_t *mod_info = __afl_module_info; mod_info; mod_info = mod_info->next) { if (mod_info->mapped) { continue; } + if (!mod_info->start) { + + fprintf(stderr, + "ERROR: __sanitizer_cov_pcs_init called with mod_info->start == " + "NULL (%s)\n", + mod_info->name); + abort(); + + } + PCTableEntry *start = (PCTableEntry *)(mod_info->pcs_beg); PCTableEntry *end = (PCTableEntry *)(mod_info->pcs_end); + if (!*mod_info->stop) { continue; } + u32 in_module_index = 0; while (start < end) { - if (mod_info->start + in_module_index >= __afl_map_size) { + if (*mod_info->start + in_module_index >= __afl_map_size) { - fprintf(stderr, "ERROR: __sanitizer_cov_pcs_init out of bounds?!\n"); + fprintf(stderr, + "ERROR: __sanitizer_cov_pcs_init out of bounds?! Start: %u " + "Stop: %u Map Size: %u (%s)\n", + *mod_info->start, *mod_info->stop, __afl_map_size, + mod_info->name); abort(); } + u32 orig_start_index = *mod_info->start; + uintptr_t PC = start->PC; // This is what `GetPreviousInstructionPc` in sanitizer runtime does @@ -1662,19 +1611,73 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, // Calculate relative offset in module PC = PC - mod_info->base_address; - __afl_pcmap_ptr[mod_info->start + in_module_index] = PC; + if (__afl_pcmap_ptr) { + + __afl_pcmap_ptr[orig_start_index + in_module_index] = PC; + + } + + if (pc_filter && !mod_info->next) { + + char PcDescr[1024]; + // This function is a part of the sanitizer run-time. + // To use it, link with AddressSanitizer or other sanitizer. + __sanitizer_symbolize_pc((void *)start->PC, "%p %F %L", PcDescr, + sizeof(PcDescr)); + + if (strstr(PcDescr, pc_filter)) { + + if (__afl_debug) + fprintf( + stderr, + "DEBUG: Selective instrumentation match: %s (PC %p Index %u)\n", + PcDescr, (void *)start->PC, + *(mod_info->start + in_module_index)); + // No change to guard needed + + } else { + + // Null out the guard to disable this edge + *(mod_info->start + in_module_index) = 0; + + } + + } + + if (__afl_filter_pcs && !mod_info->next && + strstr(mod_info->name, __afl_filter_pcs_module)) { + + u32 result_index; + if (locate_in_pcs(PC, &result_index)) { + + if (__afl_debug) + fprintf(stderr, + "DEBUG: Selective instrumentation match: (PC %lx File " + "Index %u PC Index %u)\n", + PC, result_index, in_module_index); + + } else { + + // Null out the guard to disable this edge + *(mod_info->start + in_module_index) = 0; + + } + + } start++; in_module_index++; } - mod_info->mapped = 1; + if (__afl_pcmap_ptr) { mod_info->mapped = 1; } if (__afl_debug) { - fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init initialized %u PCs\n", - in_module_index); + fprintf(stderr, + "DEBUG: __sanitizer_cov_pcs_init successfully mapped %s with %u " + "PCs\n", + mod_info->name, in_module_index); } @@ -1705,11 +1708,12 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { if (__afl_debug) { - fprintf(stderr, - "Running __sanitizer_cov_trace_pc_guard_init: %p-%p (%lu edges) " - "after_fs=%u\n", - start, stop, (unsigned long)(stop - start), - __afl_already_initialized_forkserver); + fprintf( + stderr, + "DEBUG: Running __sanitizer_cov_trace_pc_guard_init: %p-%p (%lu edges) " + "after_fs=%u *start=%u\n", + start, stop, (unsigned long)(stop - start), + __afl_already_initialized_forkserver, *start); } @@ -1741,8 +1745,8 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { mod_info->id = last_module_info ? last_module_info->id + 1 : 0; mod_info->name = strdup(dlinfo.dli_fname); mod_info->base_address = (uintptr_t)dlinfo.dli_fbase; - mod_info->start = 0; - mod_info->stop = 0; + mod_info->start = NULL; + mod_info->stop = NULL; mod_info->pcs_beg = NULL; mod_info->pcs_end = NULL; mod_info->mapped = 0; @@ -1758,8 +1762,12 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { } - fprintf(stderr, "[pcmap] Module: %s Base Address: %p\n", dlinfo.dli_fname, - dlinfo.dli_fbase); + if (__afl_debug) { + + fprintf(stderr, "[pcmap] Module: %s Base Address: %p\n", + dlinfo.dli_fname, dlinfo.dli_fbase); + + } } @@ -1807,7 +1815,8 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { u8 ignore_dso_after_fs = !!getenv("AFL_IGNORE_PROBLEMS_COVERAGE"); if (__afl_debug && ignore_dso_after_fs) { - fprintf(stderr, "Ignoring coverage from dynamically loaded code\n"); + fprintf(stderr, + "DEBUG: Ignoring coverage from dynamically loaded code\n"); } @@ -1861,12 +1870,17 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { #ifdef __AFL_CODE_COVERAGE if (mod_info) { - mod_info->start = *orig_start; - mod_info->stop = *(stop - 1); + if (!mod_info->start) { + + mod_info->start = orig_start; + mod_info->stop = stop - 1; + + } + if (__afl_debug) { fprintf(stderr, "DEBUG: [pcmap] Start Index: %u Stop Index: %u\n", - mod_info->start, mod_info->stop); + *(mod_info->start), *(mod_info->stop)); } @@ -1877,7 +1891,8 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { if (__afl_debug) { fprintf(stderr, - "Done __sanitizer_cov_trace_pc_guard_init: __afl_final_loc = %u\n", + "DEBUG: Done __sanitizer_cov_trace_pc_guard_init: __afl_final_loc " + "= %u\n", __afl_final_loc); } @@ -1888,7 +1903,7 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { if (__afl_debug) { - fprintf(stderr, "Reinit shm necessary (+%u)\n", + fprintf(stderr, "DEBUG: Reinit shm necessary (+%u)\n", __afl_final_loc - __afl_map_size); } @@ -1911,6 +1926,10 @@ void __cmplog_ins_hook1(uint8_t arg1, uint8_t arg2, uint8_t attr) { // fprintf(stderr, "hook1 arg0=%02x arg1=%02x attr=%u\n", // (u8) arg1, (u8) arg2, attr); + return; + + /* + if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); @@ -1937,11 +1956,14 @@ void __cmplog_ins_hook1(uint8_t arg1, uint8_t arg2, uint8_t attr) { __afl_cmp_map->log[k][hits].v0 = arg1; __afl_cmp_map->log[k][hits].v1 = arg2; + */ + } void __cmplog_ins_hook2(uint16_t arg1, uint16_t arg2, uint8_t attr) { - if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; + if (likely(!__afl_cmp_map)) return; + if (unlikely(arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); @@ -1979,7 +2001,8 @@ void __cmplog_ins_hook4(uint32_t arg1, uint32_t arg2, uint8_t attr) { // fprintf(stderr, "hook4 arg0=%x arg1=%x attr=%u\n", arg1, arg2, attr); - if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; + if (likely(!__afl_cmp_map)) return; + if (unlikely(arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); @@ -2017,7 +2040,8 @@ void __cmplog_ins_hook8(uint64_t arg1, uint64_t arg2, uint8_t attr) { // fprintf(stderr, "hook8 arg0=%lx arg1=%lx attr=%u\n", arg1, arg2, attr); - if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; + if (likely(!__afl_cmp_map)) return; + if (unlikely(arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); @@ -2060,7 +2084,8 @@ void __cmplog_ins_hookN(uint128_t arg1, uint128_t arg2, uint8_t attr, // (u64)(arg1 >> 64), (u64)arg1, (u64)(arg2 >> 64), (u64)arg2, size + 1, // attr); - if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; + if (likely(!__afl_cmp_map)) return; + if (unlikely(arg1 == arg2 || size > __afl_cmplog_max_len)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); @@ -2104,6 +2129,7 @@ void __cmplog_ins_hookN(uint128_t arg1, uint128_t arg2, uint8_t attr, void __cmplog_ins_hook16(uint128_t arg1, uint128_t arg2, uint8_t attr) { if (likely(!__afl_cmp_map)) return; + if (16 > __afl_cmplog_max_len) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); @@ -2143,13 +2169,13 @@ void __cmplog_ins_hook16(uint128_t arg1, uint128_t arg2, uint8_t attr) { void __sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) { - __cmplog_ins_hook1(arg1, arg2, 0); + //__cmplog_ins_hook1(arg1, arg2, 0); } void __sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2) { - __cmplog_ins_hook1(arg1, arg2, 0); + //__cmplog_ins_hook1(arg1, arg2, 0); } @@ -2258,11 +2284,13 @@ static int area_is_valid(void *ptr, size_t len) { if (unlikely(!ptr || __asan_region_is_poisoned(ptr, len))) { return 0; } -#ifndef __HAIKU__ - long r = syscall(SYS_write, __afl_dummy_fd[1], ptr, len); -#else +#ifdef __HAIKU__ long r = _kern_write(__afl_dummy_fd[1], -1, ptr, len); -#endif // HAIKU +#elif defined(__OpenBSD__) + long r = write(__afl_dummy_fd[1], ptr, len); +#else + long r = syscall(SYS_write, __afl_dummy_fd[1], ptr, len); +#endif // HAIKU, OPENBSD if (r <= 0 || r > len) return 0; @@ -2295,13 +2323,25 @@ void __cmplog_rtn_hook_strn(u8 *ptr1, u8 *ptr2, u64 len) { // fprintf(stderr, "RTN1 %p %p %u\n", ptr1, ptr2, len); if (likely(!__afl_cmp_map)) return; - if (unlikely(!len)) return; - int len0 = MIN(len, 31); + if (unlikely(!len || len > __afl_cmplog_max_len)) return; + + int len0 = MIN(len, 32); + int len1 = strnlen(ptr1, len0); - if (len1 < 31) len1 = area_is_valid(ptr1, len1 + 1); + if (len1 <= 32) len1 = area_is_valid(ptr1, len1 + 1); + if (len1 > __afl_cmplog_max_len) len1 = 0; + int len2 = strnlen(ptr2, len0); - if (len2 < 31) len2 = area_is_valid(ptr1, len2 + 1); - int l = MAX(len1, len2); + if (len2 <= 32) len2 = area_is_valid(ptr2, len2 + 1); + if (len2 > __afl_cmplog_max_len) len2 = 0; + + int l; + if (!len1) + l = len2; + else if (!len2) + l = len1; + else + l = MAX(len1, len2); if (l < 2) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); @@ -2345,10 +2385,18 @@ void __cmplog_rtn_hook_str(u8 *ptr1, u8 *ptr2) { // fprintf(stderr, "RTN1 %p %p\n", ptr1, ptr2); if (likely(!__afl_cmp_map)) return; if (unlikely(!ptr1 || !ptr2)) return; - int len1 = strnlen(ptr1, 30) + 1; - int len2 = strnlen(ptr2, 30) + 1; - int l = MAX(len1, len2); - if (l < 3) return; + int len1 = strnlen(ptr1, 31) + 1; + int len2 = strnlen(ptr2, 31) + 1; + if (len1 > __afl_cmplog_max_len) len1 = 0; + if (len2 > __afl_cmplog_max_len) len2 = 0; + int l; + if (!len1) + l = len2; + else if (!len2) + l = len1; + else + l = MAX(len1, len2); + if (l < 2) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); @@ -2390,7 +2438,7 @@ void __cmplog_rtn_hook(u8 *ptr1, u8 *ptr2) { /* u32 i; - if (area_is_valid(ptr1, 31) <= 0 || area_is_valid(ptr2, 31) <= 0) return; + if (area_is_valid(ptr1, 32) <= 0 || area_is_valid(ptr2, 32) <= 0) return; fprintf(stderr, "rtn arg0="); for (i = 0; i < 32; i++) fprintf(stderr, "%02x", ptr1[i]); @@ -2403,10 +2451,10 @@ void __cmplog_rtn_hook(u8 *ptr1, u8 *ptr2) { // fprintf(stderr, "RTN1 %p %p\n", ptr1, ptr2); if (likely(!__afl_cmp_map)) return; int l1, l2; - if ((l1 = area_is_valid(ptr1, 31)) <= 0 || - (l2 = area_is_valid(ptr2, 31)) <= 0) + if ((l1 = area_is_valid(ptr1, 32)) <= 0 || + (l2 = area_is_valid(ptr2, 32)) <= 0) return; - int len = MIN(31, MIN(l1, l2)); + int len = MIN(__afl_cmplog_max_len, MIN(l1, l2)); // fprintf(stderr, "RTN2 %u\n", len); uintptr_t k = (uintptr_t)__builtin_return_address(0); @@ -2455,7 +2503,7 @@ void __cmplog_rtn_hook_n(u8 *ptr1, u8 *ptr2, u64 len) { #if 0 /* u32 i; - if (area_is_valid(ptr1, 31) <= 0 || area_is_valid(ptr2, 31) <= 0) return; + if (area_is_valid(ptr1, 32) <= 0 || area_is_valid(ptr2, 32) <= 0) return; fprintf(stderr, "rtn_n len=%u arg0=", len); for (i = 0; i < len; i++) fprintf(stderr, "%02x", ptr1[i]); @@ -2467,12 +2515,15 @@ void __cmplog_rtn_hook_n(u8 *ptr1, u8 *ptr2, u64 len) { // fprintf(stderr, "RTN1 %p %p %u\n", ptr1, ptr2, len); if (likely(!__afl_cmp_map)) return; - if (unlikely(!len)) return; - int l = MIN(31, len); + if (!len) return; + int l = MIN(32, len), l1, l2; - if ((l = area_is_valid(ptr1, l)) <= 0 || (l = area_is_valid(ptr2, l)) <= 0) + if ((l1 = area_is_valid(ptr1, l)) <= 0 || (l2 = area_is_valid(ptr2, l)) <= 0) return; + len = MIN(l1, l2); + if (len > __afl_cmplog_max_len) return; + // fprintf(stderr, "RTN2 %u\n", l); uintptr_t k = (uintptr_t)__builtin_return_address(0); k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); @@ -2663,5 +2714,52 @@ void __afl_set_persistent_mode(u8 mode) { } +// Marker: ADD_TO_INJECTIONS + +void __afl_injection_sql(u8 *buf) { + + if (likely(buf)) { + + if (unlikely(strstr((char *)buf, "'\"\"'"))) { + + fprintf(stderr, "ALERT: Detected SQL injection in query: %s\n", buf); + abort(); + + } + + } + +} + +void __afl_injection_ldap(u8 *buf) { + + if (likely(buf)) { + + if (unlikely(strstr((char *)buf, "*)(1=*))(|"))) { + + fprintf(stderr, "ALERT: Detected LDAP injection in query: %s\n", buf); + abort(); + + } + + } + +} + +void __afl_injection_xss(u8 *buf) { + + if (likely(buf)) { + + if (unlikely(strstr((char *)buf, "1\"><\""))) { + + fprintf(stderr, "ALERT: Detected XSS injection in content: %s\n", buf); + abort(); + + } + + } + +} + #undef write_error diff --git a/instrumentation/afl-gcc-cmplog-pass.so.cc b/instrumentation/afl-gcc-cmplog-pass.so.cc index b4e6fda9..774dd5fd 100644 --- a/instrumentation/afl-gcc-cmplog-pass.so.cc +++ b/instrumentation/afl-gcc-cmplog-pass.so.cc @@ -3,7 +3,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. Copyright 2019-2020 AFLplusplus Project. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva <oliva@adacore.com>, based on the AFL++ LLVM CmpLog pass by Andrea Fioraldi <andreafioraldi@gmail.com>, and diff --git a/instrumentation/afl-gcc-cmptrs-pass.so.cc b/instrumentation/afl-gcc-cmptrs-pass.so.cc index dbb408b0..96bd5ba8 100644 --- a/instrumentation/afl-gcc-cmptrs-pass.so.cc +++ b/instrumentation/afl-gcc-cmptrs-pass.so.cc @@ -3,7 +3,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. Copyright 2019-2020 AFLplusplus Project. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva <oliva@adacore.com>, based on the AFL++ LLVM CmpLog Routines pass by Andrea Fioraldi @@ -157,6 +157,9 @@ struct afl_cmptrs_pass : afl_base_pass { /* We expect it to be a record type. */ if (TREE_CODE(t) != RECORD_TYPE) return false; + /* The type has an identifier. */ + if (!TYPE_IDENTIFIER(t)) return false; + /* The type of the template is basic_string. */ if (strcmp(IDENTIFIER_POINTER(TYPE_IDENTIFIER(t)), "basic_string") != 0) return false; @@ -177,19 +180,19 @@ struct afl_cmptrs_pass : afl_base_pass { c = DECL_CONTEXT(c); if (c && TREE_CODE(c) != TRANSLATION_UNIT_DECL) return false; - /* Check that the first nonstatic data member of the record type + /* Check that the first nonstatic named data member of the record type is named _M_dataplus. */ for (c = TYPE_FIELDS(t); c; c = DECL_CHAIN(c)) - if (TREE_CODE(c) == FIELD_DECL) break; + if (TREE_CODE(c) == FIELD_DECL && DECL_NAME(c)) break; if (!c || !integer_zerop(DECL_FIELD_BIT_OFFSET(c)) || strcmp(IDENTIFIER_POINTER(DECL_NAME(c)), "_M_dataplus") != 0) return false; - /* Check that the second nonstatic data member of the record type + /* Check that the second nonstatic named data member of the record type is named _M_string_length. */ tree f2; for (f2 = DECL_CHAIN(c); f2; f2 = DECL_CHAIN(f2)) - if (TREE_CODE(f2) == FIELD_DECL) break; + if (TREE_CODE(f2) == FIELD_DECL && DECL_NAME(f2)) break; if (!f2 /* No need to check this field's offset. */ || strcmp(IDENTIFIER_POINTER(DECL_NAME(f2)), "_M_string_length") != 0) return false; @@ -201,13 +204,16 @@ struct afl_cmptrs_pass : afl_base_pass { /* Now go back to the first data member. Its type should be a record type named _Alloc_hider. */ c = TREE_TYPE(c); - if (!c || TREE_CODE(c) != RECORD_TYPE || + if (!c || TREE_CODE(c) != RECORD_TYPE || !TYPE_IDENTIFIER(t) || strcmp(IDENTIFIER_POINTER(TYPE_IDENTIFIER(c)), "_Alloc_hider") != 0) return false; - /* And its first data member is named _M_p. */ + /* And its first nonstatic named data member should be named _M_p. + There may be (unnamed) subobjects from empty base classes. We + skip the subobjects, then check the offset of the first data + member. */ for (c = TYPE_FIELDS(c); c; c = DECL_CHAIN(c)) - if (TREE_CODE(c) == FIELD_DECL) break; + if (TREE_CODE(c) == FIELD_DECL && DECL_NAME(c)) break; if (!c || !integer_zerop(DECL_FIELD_BIT_OFFSET(c)) || strcmp(IDENTIFIER_POINTER(DECL_NAME(c)), "_M_p") != 0) return false; diff --git a/instrumentation/afl-gcc-common.h b/instrumentation/afl-gcc-common.h index 1d5eb466..80ded57d 100644 --- a/instrumentation/afl-gcc-common.h +++ b/instrumentation/afl-gcc-common.h @@ -2,7 +2,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva <oliva@adacore.com>, based on the AFL++ GCC plugin. diff --git a/instrumentation/afl-gcc-pass.so.cc b/instrumentation/afl-gcc-pass.so.cc index 4d7fd0ef..41b1e5af 100644 --- a/instrumentation/afl-gcc-pass.so.cc +++ b/instrumentation/afl-gcc-pass.so.cc @@ -2,7 +2,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva <oliva@adacore.com>, based on the AFL LLVM pass by Laszlo Szekeres <lszekeres@google.com> and Michal diff --git a/instrumentation/afl-llvm-common.cc b/instrumentation/afl-llvm-common.cc index 7f17b02d..ed9268dc 100644 --- a/instrumentation/afl-llvm-common.cc +++ b/instrumentation/afl-llvm-common.cc @@ -26,6 +26,51 @@ static std::list<std::string> allowListFunctions; static std::list<std::string> denyListFiles; static std::list<std::string> denyListFunctions; +unsigned int calcCyclomaticComplexity(llvm::Function *F) { + + unsigned int numBlocks = 0; + unsigned int numEdges = 0; + unsigned int numCalls = 0; + + // Iterate through each basic block in the function + for (BasicBlock &BB : *F) { + + // count all nodes == basic blocks + numBlocks++; + // Count the number of successors (outgoing edges) + for (BasicBlock *Succ : successors(&BB)) { + + // count edges for CC + numEdges++; + (void)(Succ); + + } + + for (Instruction &I : BB) { + + // every call is also an edge, so we need to count the calls too + if (isa<CallInst>(&I) || isa<InvokeInst>(&I)) { numCalls++; } + + } + + } + + // Cyclomatic Complexity V(G) = E - N + 2P + // For a single function, P (number of connected components) is 1 + // Calls are considered to be an edge + unsigned int CC = 2 + numCalls + numEdges - numBlocks; + + // if (debug) { + + fprintf(stderr, "CyclomaticComplexity for %s: %u\n", + F->getName().str().c_str(), CC); + + //} + + return CC; + +} + char *getBBName(const llvm::BasicBlock *BB) { static char *name; @@ -97,11 +142,15 @@ bool isIgnoreFunction(const llvm::Function *F) { static constexpr const char *ignoreSubstringList[] = { - "__asan", "__msan", "__ubsan", "__lsan", "__san", "__sanitize", - "__cxx", "DebugCounter", "DwarfDebug", "DebugLoc" + "__asan", "__msan", "__ubsan", "__lsan", "__san", + "__sanitize", "DebugCounter", "DwarfDebug", "DebugLoc" }; + // This check is very sensitive, we must be sure to not include patterns + // that are part of user-written C++ functions like the ones including + // std::string as parameter (see #1927) as the mangled type is inserted in the + // mangled name of the user-written function for (auto const &ignoreListFunc : ignoreSubstringList) { // hexcoder: F->getName().contains() not avaiilable in llvm 3.8.0 @@ -197,7 +246,7 @@ void initInstrumentList() { if (debug) DEBUGF("loaded allowlist with %zu file and %zu function entries\n", - allowListFiles.size(), allowListFunctions.size()); + allowListFiles.size() / 4, allowListFunctions.size() / 4); } @@ -272,7 +321,7 @@ void initInstrumentList() { if (debug) DEBUGF("loaded denylist with %zu file and %zu function entries\n", - denyListFiles.size(), denyListFunctions.size()); + denyListFiles.size() / 4, denyListFunctions.size() / 4); } diff --git a/instrumentation/afl-llvm-common.h b/instrumentation/afl-llvm-common.h index c9324460..6b628d64 100644 --- a/instrumentation/afl-llvm-common.h +++ b/instrumentation/afl-llvm-common.h @@ -23,7 +23,7 @@ typedef long double max_align_t; #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" #if LLVM_VERSION_MAJOR < 17 -#include "llvm/Transforms/IPO/PassManagerBuilder.h" + #include "llvm/Transforms/IPO/PassManagerBuilder.h" #endif #if LLVM_VERSION_MAJOR > 3 || \ @@ -55,6 +55,7 @@ void initInstrumentList(); bool isInInstrumentList(llvm::Function *F, std::string Filename); unsigned long long int calculateCollisions(uint32_t edges); void scanForDangerousFunctions(llvm::Module *M); +unsigned int calcCyclomaticComplexity(llvm::Function *F); #ifndef IS_EXTERN #define IS_EXTERN diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index 8ee13010..b93f61f0 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -4,7 +4,7 @@ Written by Marc Heuse <mh@mh-sec.de> - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -206,7 +206,18 @@ bool AFLdict2filePass::runOnModule(Module &M) { ptr = getenv("AFL_LLVM_DICT2FILE"); - if (!ptr || *ptr != '/') + if (!ptr) { + +#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ + auto PA = PreservedAnalyses::all(); + return PA; +#else + return true; +#endif + + } + + if (*ptr != '/') FATAL("AFL_LLVM_DICT2FILE is not set to an absolute path: %s", ptr); of.open(ptr, std::ofstream::out | std::ofstream::app); @@ -422,32 +433,35 @@ bool AFLdict2filePass::runOnModule(Module &M) { isStrstr &= FT->getNumParams() == 2 && FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == IntegerType::getInt8PtrTy(M.getContext()); + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0); isStrcmp &= FT->getNumParams() == 2 && FT->getReturnType()->isIntegerTy(32) && FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == IntegerType::getInt8PtrTy(M.getContext()); + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0); isStrcasecmp &= FT->getNumParams() == 2 && FT->getReturnType()->isIntegerTy(32) && FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == IntegerType::getInt8PtrTy(M.getContext()); + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0); isMemcmp &= FT->getNumParams() == 3 && FT->getReturnType()->isIntegerTy(32) && FT->getParamType(0)->isPointerTy() && FT->getParamType(1)->isPointerTy() && FT->getParamType(2)->isIntegerTy(); - isStrncmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()) && - FT->getParamType(2)->isIntegerTy(); - isStrncasecmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()) && - FT->getParamType(2)->isIntegerTy(); + isStrncmp &= + FT->getNumParams() == 3 && FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0) && + FT->getParamType(2)->isIntegerTy(); + isStrncasecmp &= + FT->getNumParams() == 3 && FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0) && + FT->getParamType(2)->isIntegerTy(); isStdString &= FT->getNumParams() >= 2 && FT->getParamType(0)->isPointerTy() && FT->getParamType(1)->isPointerTy(); @@ -732,7 +746,7 @@ bool AFLdict2filePass::runOnModule(Module &M) { auto PA = PreservedAnalyses::all(); return PA; #else - return true; + return false; #endif } diff --git a/instrumentation/afl-llvm-lto-instrumentlist.so.cc b/instrumentation/afl-llvm-lto-instrumentlist.so.cc index db5bd55e..e0899cd3 100644 --- a/instrumentation/afl-llvm-lto-instrumentlist.so.cc +++ b/instrumentation/afl-llvm-lto-instrumentlist.so.cc @@ -9,7 +9,7 @@ from afl-as.c are Michal's fault. Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ #include "llvm/IR/Module.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" -//#include "llvm/Transforms/IPO/PassManagerBuilder.h" +// #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/IR/PassManager.h" diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index c59324fd..75b8532b 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -12,7 +12,7 @@ NGRAM previous location coverage comes from Adrian Herrera. Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -128,7 +128,11 @@ llvmGetPassPluginInfo() { #if LLVM_VERSION_MAJOR <= 13 using OptimizationLevel = typename PassBuilder::OptimizationLevel; #endif + #if LLVM_VERSION_MAJOR >= 16 + PB.registerOptimizerEarlyEPCallback( + #else PB.registerOptimizerLastEPCallback( + #endif [](ModulePassManager &MPM, OptimizationLevel OL) { MPM.addPass(AFLCoverage()); @@ -212,10 +216,6 @@ bool AFLCoverage::runOnModule(Module &M) { u32 rand_seed; unsigned int cur_loc = 0; -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - auto PA = PreservedAnalyses::all(); -#endif - /* Setup random() so we get Actually Random(TM) outputs from AFL_R() */ gettimeofday(&tv, &tz); rand_seed = tv.tv_sec ^ tv.tv_usec ^ getpid(); @@ -552,7 +552,7 @@ bool AFLCoverage::runOnModule(Module &M) { #endif { - // load the context ID of the previous function and write to to a + // load the context ID of the previous function and write to a // local variable on the stack LoadInst *PrevCtxLoad = IRB.CreateLoad( #if LLVM_VERSION_MAJOR >= 14 @@ -634,7 +634,7 @@ bool AFLCoverage::runOnModule(Module &M) { /* There is a problem with Ubuntu 18.04 and llvm 6.0 (see issue #63). The inline function successors() is not inlined and also not found at runtime - :-( As I am unable to detect Ubuntu18.04 heree, the next best thing is to + :-( As I am unable to detect Ubuntu18.04 here, the next best thing is to disable this optional optimization for LLVM 6.0.0 and Linux */ #if !(LLVM_VERSION_MAJOR == 6 && LLVM_VERSION_MINOR == 0) || !defined __linux__ // only instrument if this basic block is the destination of a previous @@ -1081,7 +1081,7 @@ bool AFLCoverage::runOnModule(Module &M) { } #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - return PA; + return PreservedAnalyses(); #else return true; #endif diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index bca1f927..fe5c2926 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -5,7 +5,7 @@ Written by Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -90,7 +90,7 @@ class CmpLogInstructions : public ModulePass { #if LLVM_MAJOR >= 11 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); #else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR >= 4 StringRef getPassName() const override { @@ -165,23 +165,25 @@ bool CmpLogInstructions::hookInstrs(Module &M) { IntegerType *Int64Ty = IntegerType::getInt64Ty(C); IntegerType *Int128Ty = IntegerType::getInt128Ty(C); -#if LLVM_VERSION_MAJOR >= 9 - FunctionCallee -#else - Constant * -#endif - c1 = M.getOrInsertFunction("__cmplog_ins_hook1", VoidTy, Int8Ty, Int8Ty, - Int8Ty -#if LLVM_VERSION_MAJOR < 5 - , - NULL -#endif - ); -#if LLVM_VERSION_MAJOR >= 9 - FunctionCallee cmplogHookIns1 = c1; -#else - Function *cmplogHookIns1 = cast<Function>(c1); -#endif + /* + #if LLVM_VERSION_MAJOR >= 9 + FunctionCallee + #else + Constant * + #endif + c1 = M.getOrInsertFunction("__cmplog_ins_hook1", VoidTy, Int8Ty, Int8Ty, + Int8Ty + #if LLVM_VERSION_MAJOR < 5 + , + NULL + #endif + ); + #if LLVM_VERSION_MAJOR >= 9 + FunctionCallee cmplogHookIns1 = c1; + #else + Function *cmplogHookIns1 = cast<Function>(c1); + #endif + */ #if LLVM_VERSION_MAJOR >= 9 FunctionCallee @@ -619,7 +621,7 @@ bool CmpLogInstructions::hookInstrs(Module &M) { switch (cast_size) { case 8: - IRB.CreateCall(cmplogHookIns1, args); + // IRB.CreateCall(cmplogHookIns1, args); break; case 16: IRB.CreateCall(cmplogHookIns2, args); @@ -678,13 +680,16 @@ bool CmpLogInstructions::runOnModule(Module &M) { printf("Running cmplog-instructions-pass by andreafioraldi@gmail.com\n"); else be_quiet = 1; - hookInstrs(M); + bool ret = hookInstrs(M); verifyModule(M); #if LLVM_MAJOR >= 11 /* use new pass manager */ - return PreservedAnalyses::all(); + if (ret == false) + return PreservedAnalyses::all(); + else + return PreservedAnalyses(); #else - return true; + return ret; #endif } diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index 39db5aa4..560bd73b 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -5,7 +5,7 @@ Written by Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -85,7 +85,7 @@ class CmpLogRoutines : public ModulePass { #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); #else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR >= 4 StringRef getPassName() const override { @@ -385,7 +385,8 @@ bool CmpLogRoutines::hookRtns(Module &M) { isStrcmp &= FT->getNumParams() == 2 && FT->getReturnType()->isIntegerTy(32) && FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == IntegerType::getInt8PtrTy(M.getContext()); + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0); bool isStrncmp = (!FuncName.compare("strncmp") || !FuncName.compare("xmlStrncmp") || @@ -398,12 +399,12 @@ bool CmpLogRoutines::hookRtns(Module &M) { !FuncName.compare("g_ascii_strncasecmp") || !FuncName.compare("Curl_strncasecompare") || !FuncName.compare("g_strncasecmp")); - isStrncmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()) && - FT->getParamType(2)->isIntegerTy(); + isStrncmp &= + FT->getNumParams() == 3 && FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0) && + FT->getParamType(2)->isIntegerTy(); bool isGccStdStringStdString = Callee->getName().find("__is_charIT_EE7__value") != @@ -542,7 +543,7 @@ bool CmpLogRoutines::hookRtns(Module &M) { Value *v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); Value *v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); Value *v3Pbitcast = IRB.CreateBitCast( - v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); + v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); Value *v3Pcasted = IRB.CreateIntCast(v3Pbitcast, IntegerType::get(C, 64), false); args.push_back(v1Pcasted); @@ -608,7 +609,7 @@ bool CmpLogRoutines::hookRtns(Module &M) { Value *v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); Value *v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); Value *v3Pbitcast = IRB.CreateBitCast( - v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); + v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); Value *v3Pcasted = IRB.CreateIntCast(v3Pbitcast, IntegerType::get(C, 64), false); args.push_back(v1Pcasted); @@ -757,16 +758,16 @@ bool CmpLogRoutines::runOnModule(Module &M) { printf("Running cmplog-routines-pass by andreafioraldi@gmail.com\n"); else be_quiet = 1; - hookRtns(M); -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - auto PA = PreservedAnalyses::all(); -#endif + bool ret = hookRtns(M); verifyModule(M); #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - return PA; + if (ret == false) + return PreservedAnalyses::all(); + else + return PreservedAnalyses(); #else - return true; + return ret; #endif } diff --git a/instrumentation/cmplog-switches-pass.cc b/instrumentation/cmplog-switches-pass.cc index 38de669d..2b87ea8c 100644 --- a/instrumentation/cmplog-switches-pass.cc +++ b/instrumentation/cmplog-switches-pass.cc @@ -5,7 +5,7 @@ Written by Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -85,7 +85,7 @@ class CmplogSwitches : public ModulePass { #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); #else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR < 4 const char *getPassName() const override { @@ -442,16 +442,16 @@ bool CmplogSwitches::runOnModule(Module &M) { printf("Running cmplog-switches-pass by andreafioraldi@gmail.com\n"); else be_quiet = 1; - hookInstrs(M); -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - auto PA = PreservedAnalyses::all(); -#endif + bool ret = hookInstrs(M); verifyModule(M); #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - return PA; + if (ret == false) + return PreservedAnalyses::all(); + else + return PreservedAnalyses(); #else - return true; + return ret; #endif } diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index efc99d20..496d69fc 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -89,7 +89,7 @@ class CompareTransform : public ModulePass { #endif - return "cmplog transform"; + return "compcov transform"; } @@ -123,7 +123,11 @@ llvmGetPassPluginInfo() { #if LLVM_VERSION_MAJOR <= 13 using OptimizationLevel = typename PassBuilder::OptimizationLevel; #endif + #if LLVM_VERSION_MAJOR >= 16 + PB.registerOptimizerEarlyEPCallback( + #else PB.registerOptimizerLastEPCallback( + #endif [](ModulePassManager &MPM, OptimizationLevel OL) { MPM.addPass(CompareTransform()); @@ -169,6 +173,7 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, DenseMap<Value *, std::string *> valueMap; std::vector<CallInst *> calls; LLVMContext &C = M.getContext(); + IntegerType *Int1Ty = IntegerType::getInt1Ty(C); IntegerType *Int8Ty = IntegerType::getInt8Ty(C); IntegerType *Int32Ty = IntegerType::getInt32Ty(C); IntegerType *Int64Ty = IntegerType::getInt64Ty(C); @@ -225,38 +230,38 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, if (callInst->getCallingConv() != llvm::CallingConv::C) continue; StringRef FuncName = Callee->getName(); isStrcmp &= - (!FuncName.compare("strcmp") || !FuncName.compare("xmlStrcmp") || + (!FuncName.compare("strcmp") /*|| !FuncName.compare("xmlStrcmp") || !FuncName.compare("xmlStrEqual") || - !FuncName.compare("g_strcmp0") || !FuncName.compare("curl_strequal") || - !FuncName.compare("strcsequal")); + !FuncName.compare("strcsequal") || + !FuncName.compare("g_strcmp0")*/); isMemcmp &= (!FuncName.compare("memcmp") || !FuncName.compare("bcmp") || !FuncName.compare("CRYPTO_memcmp") || !FuncName.compare("OPENSSL_memcmp") || !FuncName.compare("memcmp_const_time") || !FuncName.compare("memcmpct")); - isStrncmp &= (!FuncName.compare("strncmp") || - !FuncName.compare("xmlStrncmp") || - !FuncName.compare("curl_strnequal")); + isStrncmp &= (!FuncName.compare("strncmp")/* || + !FuncName.compare("curl_strnequal") || + !FuncName.compare("xmlStrncmp")*/); isStrcasecmp &= (!FuncName.compare("strcasecmp") || !FuncName.compare("stricmp") || !FuncName.compare("ap_cstr_casecmp") || !FuncName.compare("OPENSSL_strcasecmp") || - !FuncName.compare("xmlStrcasecmp") || + /*!FuncName.compare("xmlStrcasecmp") || !FuncName.compare("g_strcasecmp") || !FuncName.compare("g_ascii_strcasecmp") || !FuncName.compare("Curl_strcasecompare") || - !FuncName.compare("Curl_safe_strcasecompare") || + !FuncName.compare("Curl_safe_strcasecompare") ||*/ !FuncName.compare("cmsstrcasecmp")); isStrncasecmp &= (!FuncName.compare("strncasecmp") || !FuncName.compare("strnicmp") || !FuncName.compare("ap_cstr_casecmpn") || - !FuncName.compare("OPENSSL_strncasecmp") || + !FuncName.compare("OPENSSL_strncasecmp") /*|| !FuncName.compare("xmlStrncasecmp") || !FuncName.compare("g_ascii_strncasecmp") || !FuncName.compare("Curl_strncasecompare") || - !FuncName.compare("g_strncasecmp")); + !FuncName.compare("g_strncasecmp")*/); isIntMemcpy &= !FuncName.compare("llvm.memcpy.p0i8.p0i8.i64"); if (!isStrcmp && !isMemcmp && !isStrncmp && !isStrcasecmp && @@ -270,28 +275,30 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, isStrcmp &= FT->getNumParams() == 2 && FT->getReturnType()->isIntegerTy(32) && FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == IntegerType::getInt8PtrTy(M.getContext()); + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0); isStrcasecmp &= FT->getNumParams() == 2 && FT->getReturnType()->isIntegerTy(32) && FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == IntegerType::getInt8PtrTy(M.getContext()); + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0); isMemcmp &= FT->getNumParams() == 3 && FT->getReturnType()->isIntegerTy(32) && FT->getParamType(0)->isPointerTy() && FT->getParamType(1)->isPointerTy() && FT->getParamType(2)->isIntegerTy(); - isStrncmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()) && - FT->getParamType(2)->isIntegerTy(); - isStrncasecmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()) && - FT->getParamType(2)->isIntegerTy(); + isStrncmp &= + FT->getNumParams() == 3 && FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0) && + FT->getParamType(2)->isIntegerTy(); + isStrncasecmp &= + FT->getNumParams() == 3 && FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8Ty(M.getContext())->getPointerTo(0) && + FT->getParamType(2)->isIntegerTy(); if (!isStrcmp && !isMemcmp && !isStrncmp && !isStrcasecmp && !isStrncasecmp && !isIntMemcpy) @@ -457,8 +464,21 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, bool isSizedcmp = false; bool isCaseInsensitive = false; bool needs_null = false; + bool success_is_one = false; + bool nullCheck = false; Function *Callee = callInst->getCalledFunction(); + /* + fprintf(stderr, "%s - %s - %s\n", + callInst->getParent() + ->getParent() + ->getParent() + ->getName() + .str() + .c_str(), + callInst->getParent()->getParent()->getName().str().c_str(), + Callee ? Callee->getName().str().c_str() : "NULL");*/ + if (Callee) { if (!Callee->getName().compare("memcmp") || @@ -503,9 +523,20 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, !Callee->getName().compare("g_strncasecmp")) isCaseInsensitive = true; + if (!Callee->getName().compare("xmlStrEqual") || + !Callee->getName().compare("curl_strequal") || + !Callee->getName().compare("strcsequal") || + !Callee->getName().compare("curl_strnequal")) + success_is_one = true; + } if (!isSizedcmp) needs_null = true; + if (Callee->getName().startswith("g_") || + Callee->getName().startswith("curl_") || + Callee->getName().startswith("Curl_") || + Callee->getName().startswith("xml")) + nullCheck = true; Value *sizedValue = isSizedcmp ? callInst->getArgOperand(2) : NULL; bool isConstSized = sizedValue && isa<ConstantInt>(sizedValue); @@ -590,8 +621,10 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, /* split before the call instruction */ BasicBlock *bb = callInst->getParent(); BasicBlock *end_bb = bb->splitBasicBlock(BasicBlock::iterator(callInst)); - BasicBlock *next_lenchk_bb = NULL; + + if (nullCheck) { fprintf(stderr, "TODO: null check\n"); } + if (isSizedcmp && !isConstSized) { next_lenchk_bb = @@ -623,7 +656,7 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, IRBuilder<> cur_lenchk_IRB(&*(cur_lenchk_bb->getFirstInsertionPt())); Value *icmp = cur_lenchk_IRB.CreateICmpEQ( - sizedValue, ConstantInt::get(sizedValue->getType(), i)); + sizedValue, ConstantInt::get(sizedValue->getType(), i)); cur_lenchk_IRB.CreateCondBr(icmp, end_bb, cur_cmp_bb); cur_lenchk_bb->getTerminator()->eraseFromParent(); @@ -667,6 +700,14 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, else isub = cur_cmp_IRB.CreateSub(load, ConstantInt::get(Int8Ty, c)); + if (success_is_one && i == unrollLen - 1) { + + Value *isubsub = cur_cmp_IRB.CreateTrunc(isub, Int1Ty); + isub = cur_cmp_IRB.CreateSelect(isubsub, ConstantInt::get(Int8Ty, 0), + ConstantInt::get(Int8Ty, 1)); + + } + Value *sext = cur_cmp_IRB.CreateSExt(isub, Int32Ty); PN->addIncoming(sext, cur_cmp_bb); @@ -728,6 +769,8 @@ bool CompareTransform::runOnModule(Module &M) { #endif + bool ret = false; + if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) printf( "Running compare-transform-pass by laf.intel@gmail.com, extended by " @@ -735,11 +778,7 @@ bool CompareTransform::runOnModule(Module &M) { else be_quiet = 1; -#if LLVM_MAJOR >= 11 /* use new pass manager */ - auto PA = PreservedAnalyses::all(); -#endif - - transformCmps(M, true, true, true, true, true); + if (transformCmps(M, true, true, true, true, true) == true) ret = true; verifyModule(M); #if LLVM_MAJOR >= 11 /* use new pass manager */ @@ -749,9 +788,18 @@ bool CompareTransform::runOnModule(Module &M) { }*/ - return PA; + if (ret == true) { + + return PreservedAnalyses(); + + } else { + + return PreservedAnalyses::all(); + + } + #else - return true; + return ret; #endif } diff --git a/instrumentation/injection-pass.cc b/instrumentation/injection-pass.cc new file mode 100644 index 00000000..47ddabd9 --- /dev/null +++ b/instrumentation/injection-pass.cc @@ -0,0 +1,369 @@ +/* + american fuzzy lop++ - LLVM Injection instrumentation + -------------------------------------------------- + + Written by Marc Heuse <mh@mh-sec.de> + + Copyright 2015, 2016 Google Inc. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at: + + https://www.apache.org/licenses/LICENSE-2.0 + +*/ + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include <list> +#include <string> +#include <fstream> +#include <sys/time.h> +#include "llvm/Config/llvm-config.h" + +#include "llvm/ADT/Statistic.h" +#include "llvm/IR/IRBuilder.h" +#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ + #include "llvm/Passes/PassPlugin.h" + #include "llvm/Passes/PassBuilder.h" + #include "llvm/IR/PassManager.h" +#else + #include "llvm/IR/LegacyPassManager.h" + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif +#include "llvm/IR/Module.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Pass.h" +#include "llvm/Analysis/ValueTracking.h" + +#include "llvm/IR/IRBuilder.h" +#if LLVM_VERSION_MAJOR >= 4 || \ + (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) + #include "llvm/IR/Verifier.h" + #include "llvm/IR/DebugInfo.h" +#else + #include "llvm/Analysis/Verifier.h" + #include "llvm/DebugInfo.h" + #define nullptr 0 +#endif + +#include <set> +#include "afl-llvm-common.h" + +using namespace llvm; + +namespace { + +#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ +class InjectionRoutines : public PassInfoMixin<InjectionRoutines> { + + public: + InjectionRoutines() { + +#else +class InjectionRoutines : public ModulePass { + + public: + static char ID; + InjectionRoutines() : ModulePass(ID) { + +#endif + + initInstrumentList(); + + } + +#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +#else + bool runOnModule(Module &M) override; + + #if LLVM_VERSION_MAJOR >= 4 + StringRef getPassName() const override { + + #else + const char *getPassName() const override { + + #endif + return "Injection routines"; + + } + +#endif + + private: + bool hookRtns(Module &M); + + bool doSQL = false; + bool doLDAP = false; + bool doXSS = false; + +}; + +} // namespace + +#if LLVM_MAJOR >= 11 +extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK +llvmGetPassPluginInfo() { + + return {LLVM_PLUGIN_API_VERSION, "Injectionroutines", "v0.1", + /* lambda to insert our pass into the pass pipeline. */ + [](PassBuilder &PB) { + + #if LLVM_VERSION_MAJOR <= 13 + using OptimizationLevel = typename PassBuilder::OptimizationLevel; + #endif + PB.registerOptimizerLastEPCallback( + [](ModulePassManager &MPM, OptimizationLevel OL) { + + MPM.addPass(InjectionRoutines()); + + }); + + }}; + +} + +#else +char InjectionRoutines::ID = 0; +#endif + +bool InjectionRoutines::hookRtns(Module &M) { + + std::vector<CallInst *> calls, llvmStdStd, llvmStdC, gccStdStd, gccStdC, + Memcmp, Strcmp, Strncmp; + LLVMContext &C = M.getContext(); + + Type *VoidTy = Type::getVoidTy(C); + IntegerType *Int8Ty = IntegerType::getInt8Ty(C); + PointerType *i8PtrTy = PointerType::get(Int8Ty, 0); + +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee +#else + Constant * +#endif + c1 = M.getOrInsertFunction("__afl_injection_sql", VoidTy, i8PtrTy +#if LLVM_VERSION_MAJOR < 5 + , + NULL +#endif + ); +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee sqlfunc = c1; +#else + Function *sqlfunc = cast<Function>(c1); +#endif + +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee +#else + Constant * +#endif + c2 = M.getOrInsertFunction("__afl_injection_ldap", VoidTy, i8PtrTy +#if LLVM_VERSION_MAJOR < 5 + , + NULL +#endif + ); +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee ldapfunc = c2; +#else + Function *ldapfunc = cast<Function>(c2); +#endif + +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee +#else + Constant * +#endif + c3 = M.getOrInsertFunction("__afl_injection_xss", VoidTy, i8PtrTy +#if LLVM_VERSION_MAJOR < 5 + , + NULL +#endif + ); +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee xssfunc = c3; +#else + Function *xssfunc = cast<Function>(c3); +#endif + +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee FuncPtr; +#else + Function *FuncPtr; +#endif + + bool ret = false; + + /* iterate over all functions, bbs and instruction and add suitable calls */ + for (auto &F : M) { + + if (!isInInstrumentList(&F, MNAME)) continue; + + for (auto &BB : F) { + + for (auto &IN : BB) { + + CallInst *callInst = nullptr; + + if ((callInst = dyn_cast<CallInst>(&IN))) { + + Function *Callee = callInst->getCalledFunction(); + if (!Callee) continue; + if (callInst->getCallingConv() != llvm::CallingConv::C) continue; + + std::string FuncName = Callee->getName().str(); + FuncPtr = nullptr; + size_t param = 0; + + // Marker: ADD_TO_INJECTIONS + // If you just need to add another function to test for SQL etc. + // then add them here. + // To add a new class or to work on e.g. std::string/Rust strings/... + // you will need to add a function to afl-compiler-rt.c.o and + // and upwards in this file add a pointer to that function to use + // here. + + if (doSQL && + (FuncName.compare("sqlite3_exec") == 0 || + FuncName.compare("PQexec") == 0 || FuncName.compare("") == 0 || + FuncName.compare("PQexecParams") == 0 || + FuncName.compare("mysql_query") == 0)) { + + if (!be_quiet) { + + errs() << "Injection SQL hook: " << FuncName << "\n"; + + } + + FuncPtr = sqlfunc; + param = 1; + + } + + if (doLDAP && (FuncName.compare("ldap_search_ext") == 0 || + FuncName.compare("ldap_search_ext_s") == 0)) { + + if (!be_quiet) { + + errs() << "Injection LDAP hook: " << FuncName << "\n"; + + } + + FuncPtr = ldapfunc; + param = 1; + + } + + if (doXSS && (FuncName.compare("htmlReadMemory") == 0)) { + + if (!be_quiet) { + + errs() << "Injection XSS hook: " << FuncName << "\n"; + + } + + FuncPtr = xssfunc; + param = 1; + + } + + if (FuncPtr) { + + IRBuilder<> IRB(callInst->getParent()); + IRB.SetInsertPoint(callInst); + ret = true; + + Value *parameter = callInst->getArgOperand(param); + + std::vector<Value *> args; + Value *casted = IRB.CreatePointerCast(parameter, i8PtrTy); + args.push_back(casted); + IRB.CreateCall(FuncPtr, args); + + } + + } + + } + + } + + } + + return ret; + +} + +#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ +PreservedAnalyses InjectionRoutines::run(Module &M, + ModuleAnalysisManager &MAM) { + +#else +bool InjectionRoutines::runOnModule(Module &M) { + +#endif + + if (getenv("AFL_QUIET") == NULL) + printf("Running injection-pass by Marc Heuse (mh@mh-sec.de)\n"); + else + be_quiet = 1; + if (getenv("AFL_LLVM_INJECTIONS_ALL")) { + + doSQL = true; + doLDAP = true; + doXSS = true; + + } + + if (getenv("AFL_LLVM_INJECTIONS_SQL")) { doSQL = true; } + if (getenv("AFL_LLVM_INJECTIONS_LDAP")) { doLDAP = true; } + if (getenv("AFL_LLVM_INJECTIONS_XSS")) { doXSS = true; } + + bool ret = hookRtns(M); + verifyModule(M); + +#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ + if (ret == false) + return PreservedAnalyses::all(); + else + return PreservedAnalyses(); +#else + return ret; +#endif + +} + +#if LLVM_VERSION_MAJOR < 11 /* use old pass manager */ +static void registerInjectionRoutinesPass(const PassManagerBuilder &, + legacy::PassManagerBase &PM) { + + auto p = new InjectionRoutines(); + PM.add(p); + +} + +static RegisterStandardPasses RegisterInjectionRoutinesPass( + PassManagerBuilder::EP_OptimizerLast, registerInjectionRoutinesPass); + +static RegisterStandardPasses RegisterInjectionRoutinesPass0( + PassManagerBuilder::EP_EnabledOnOptLevel0, registerInjectionRoutinesPass); + + #if LLVM_VERSION_MAJOR >= 11 +static RegisterStandardPasses RegisterInjectionRoutinesPassLTO( + PassManagerBuilder::EP_FullLinkTimeOptimizationLast, + registerInjectionRoutinesPass); + #endif +#endif + diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index 8a07610c..effafe50 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -1,7 +1,7 @@ /* * Copyright 2016 laf-intel - * extended for floating point by Heiko EiĆfeldt - * adapted to new pass manager by Heiko EiĆfeldt + * extended for floating point by Heiko Eissfeldt + * adapted to new pass manager by Heiko Eissfeldt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -60,7 +60,7 @@ using namespace llvm; // uncomment this toggle function verification at each step. horribly slow, but // helps to pinpoint a potential problem in the splitting code. -//#define VERIFY_TOO_MUCH 1 +// #define VERIFY_TOO_MUCH 1 namespace { @@ -189,7 +189,11 @@ llvmGetPassPluginInfo() { #if LLVM_VERSION_MAJOR <= 13 using OptimizationLevel = typename PassBuilder::OptimizationLevel; #endif + #if LLVM_VERSION_MAJOR >= 16 + PB.registerOptimizerEarlyEPCallback( + #else PB.registerOptimizerLastEPCallback( + #endif [](ModulePassManager &MPM, OptimizationLevel OL) { MPM.addPass(SplitComparesTransform()); @@ -262,8 +266,11 @@ bool SplitComparesTransform::simplifyFPCompares(Module &M) { /* this is probably not needed but we do it anyway */ if (TyOp0 != TyOp1) { continue; } - if (TyOp0->isArrayTy() || TyOp0->isVectorTy()) { continue; } + int constants = 0; + if (llvm::isa<llvm::Constant>(op0)) { ++constants; } + if (llvm::isa<llvm::Constant>(op1)) { ++constants; } + if (constants != 1) { continue; } fcomps.push_back(selectcmpInst); @@ -463,8 +470,12 @@ bool SplitComparesTransform::simplifyOrEqualsCompare(CmpInst *IcmpInst, #else ReplaceInstWithInst(IcmpInst->getParent()->getInstList(), ii, PN); #endif + if (new_pred == CmpInst::ICMP_SGT || new_pred == CmpInst::ICMP_SLT) { + + simplifySignedCompare(icmp_np, M, worklist); + + } - worklist.push_back(icmp_np); worklist.push_back(icmp_eq); return true; @@ -740,17 +751,24 @@ bool SplitComparesTransform::splitCompare(CmpInst *cmp_inst, Module &M, CmpInst *icmp_inv_cmp = nullptr; BasicBlock *inv_cmp_bb = BasicBlock::Create(C, "inv_cmp", end_bb->getParent(), end_bb); - if (pred == CmpInst::ICMP_UGT || pred == CmpInst::ICMP_SGT || - pred == CmpInst::ICMP_UGE || pred == CmpInst::ICMP_SGE) { + if (pred == CmpInst::ICMP_UGT) { icmp_inv_cmp = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULT, op0_high, op1_high); - } else { + } else if (pred == CmpInst::ICMP_ULT) { icmp_inv_cmp = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_UGT, op0_high, op1_high); + } else { + + // Never gonna appen + if (!be_quiet) + fprintf(stderr, + "Error: split-compare: Equals or signed not removed: %d\n", + pred); + } #if LLVM_MAJOR >= 16 @@ -924,7 +942,7 @@ size_t SplitComparesTransform::nextPowerOfTwo(size_t in) { /* splits fcmps into two nested fcmps with sign compare and the rest */ size_t SplitComparesTransform::splitFPCompares(Module &M) { - size_t count = 0; + size_t counts = 0; LLVMContext &C = M.getContext(); @@ -940,7 +958,7 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { } else { - return count; + return counts; } @@ -993,7 +1011,7 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { } - if (!fcomps.size()) { return count; } + if (!fcomps.size()) { return counts; } IntegerType *Int1Ty = IntegerType::getInt1Ty(C); @@ -1573,7 +1591,7 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_UGT, t_f0, t_f1); #if LLVM_MAJOR >= 16 icmp_fraction_result->insertInto(negative_bb, negative_bb->end()); - icmp_fraction_result2->insertInto(positive_bb, negative_bb->end()); + icmp_fraction_result2->insertInto(positive_bb, positive_bb->end()); #else negative_bb->getInstList().push_back(icmp_fraction_result); positive_bb->getInstList().push_back(icmp_fraction_result2); @@ -1587,7 +1605,7 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULT, t_f0, t_f1); #if LLVM_MAJOR >= 16 icmp_fraction_result->insertInto(negative_bb, negative_bb->end()); - icmp_fraction_result2->insertInto(positive_bb, negative_bb->end()); + icmp_fraction_result2->insertInto(positive_bb, positive_bb->end()); #else negative_bb->getInstList().push_back(icmp_fraction_result); positive_bb->getInstList().push_back(icmp_fraction_result2); @@ -1679,11 +1697,11 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { #else ReplaceInstWithInst(FcmpInst->getParent()->getInstList(), ii, PN); #endif - ++count; + ++counts; } - return count; + return counts; } @@ -1696,12 +1714,6 @@ bool SplitComparesTransform::runOnModule(Module &M) { #endif - char *bitw_env = getenv("AFL_LLVM_LAF_SPLIT_COMPARES_BITW"); - if (!bitw_env) bitw_env = getenv("LAF_SPLIT_COMPARES_BITW"); - if (bitw_env) { target_bitwidth = atoi(bitw_env); } - - enableFPSplit = getenv("AFL_LLVM_LAF_SPLIT_FLOATS") != NULL; - if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) { @@ -1717,10 +1729,27 @@ bool SplitComparesTransform::runOnModule(Module &M) { } -#if LLVM_MAJOR >= 11 - auto PA = PreservedAnalyses::all(); + char *bitw_env = getenv("AFL_LLVM_LAF_SPLIT_COMPARES_BITW"); + if (!bitw_env) bitw_env = getenv("LAF_SPLIT_COMPARES_BITW"); + if (bitw_env) { target_bitwidth = atoi(bitw_env); } + + if (getenv("AFL_LLVM_LAF_SPLIT_FLOATS")) { enableFPSplit = true; } + + bool split_comp = false; + + if (getenv("AFL_LLVM_LAF_SPLIT_COMPARES")) { + +#if LLVM_MAJOR == 17 + if (!be_quiet) + fprintf(stderr, + "WARNING: AFL++ splitting integer comparisons is disabled in " + "LLVM 17 due bugs, switch to 16 or 18!\n"); +#else + split_comp = true; #endif + } + if (enableFPSplit) { simplifyFPCompares(M); @@ -1729,42 +1758,44 @@ bool SplitComparesTransform::runOnModule(Module &M) { if (!be_quiet && !debug) { errs() << "Split-floatingpoint-compare-pass: " << count - << " FP comparisons splitted\n"; + << " FP comparisons split\n"; } } - std::vector<CmpInst *> worklist; - /* iterate over all functions, bbs and instruction search for all integer - * compare instructions. Save them into the worklist for later. */ - for (auto &F : M) { + if (split_comp) { - if (!isInInstrumentList(&F, MNAME)) continue; + std::vector<CmpInst *> worklist; + /* iterate over all functions, bbs and instruction search for all integer + * compare instructions. Save them into the worklist for later. */ + for (auto &F : M) { - for (auto &BB : F) { + if (!isInInstrumentList(&F, MNAME)) continue; - for (auto &IN : BB) { + for (auto &BB : F) { - if (auto CI = dyn_cast<CmpInst>(&IN)) { + for (auto &IN : BB) { - auto op0 = CI->getOperand(0); - auto op1 = CI->getOperand(1); - if (!op0 || !op1) { + if (auto CI = dyn_cast<CmpInst>(&IN)) { -#if LLVM_MAJOR >= 11 - return PA; -#else - return false; -#endif + auto op0 = CI->getOperand(0); + auto op1 = CI->getOperand(1); + // has to valid operands + if (!op0 || !op1) { continue; } + // has exactly one constant and one variable + int constants = 0; + if (dyn_cast<ConstantInt>(op0)) { ++constants; } + if (dyn_cast<ConstantInt>(op1)) { ++constants; } + if (constants != 1) { continue; } - } + auto iTy1 = dyn_cast<IntegerType>(op0->getType()); + if (iTy1 && isa<IntegerType>(op1->getType())) { - auto iTy1 = dyn_cast<IntegerType>(op0->getType()); - if (iTy1 && isa<IntegerType>(op1->getType())) { + unsigned bitw = iTy1->getBitWidth(); + if (isSupportedBitWidth(bitw)) { worklist.push_back(CI); } - unsigned bitw = iTy1->getBitWidth(); - if (isSupportedBitWidth(bitw)) { worklist.push_back(CI); } + } } @@ -1774,16 +1805,18 @@ bool SplitComparesTransform::runOnModule(Module &M) { } - } + // now that we have a list of all integer comparisons we can start replacing + // them with the splitted alternatives. + for (auto CI : worklist) { - // now that we have a list of all integer comparisons we can start replacing - // them with the splitted alternatives. - for (auto CI : worklist) { + simplifyAndSplit(CI, M); - simplifyAndSplit(CI, M); + } } + bool ret = count == 0 ? false : true; + bool brokenDebug = false; if (verifyModule(M, &errs() #if LLVM_VERSION_MAJOR >= 4 || \ @@ -1822,9 +1855,12 @@ bool SplitComparesTransform::runOnModule(Module &M) { }*/ - return PA; + if (ret == false) + return PreservedAnalyses::all(); + else + return PreservedAnalyses(); #else - return true; + return ret; #endif } diff --git a/instrumentation/split-switches-pass.so.cc b/instrumentation/split-switches-pass.so.cc index dcd89652..aa552a42 100644 --- a/instrumentation/split-switches-pass.so.cc +++ b/instrumentation/split-switches-pass.so.cc @@ -84,7 +84,7 @@ class SplitSwitchesTransform : public ModulePass { #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); #else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR >= 4 StringRef getPassName() const override { @@ -137,7 +137,11 @@ llvmGetPassPluginInfo() { #if LLVM_VERSION_MAJOR <= 13 using OptimizationLevel = typename PassBuilder::OptimizationLevel; #endif + #if LLVM_VERSION_MAJOR >= 16 + PB.registerOptimizerEarlyEPCallback( + #else PB.registerOptimizerLastEPCallback( + #endif [](ModulePassManager &MPM, OptimizationLevel OL) { MPM.addPass(SplitSwitchesTransform()); @@ -516,11 +520,7 @@ bool SplitSwitchesTransform::runOnModule(Module &M) { else be_quiet = 1; -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - auto PA = PreservedAnalyses::all(); -#endif - - splitSwitches(M); + bool ret = splitSwitches(M); verifyModule(M); #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ @@ -530,9 +530,12 @@ bool SplitSwitchesTransform::runOnModule(Module &M) { }*/ - return PA; + if (ret == false) + return PreservedAnalyses::all(); + else + return PreservedAnalyses(); #else - return true; + return ret; #endif } diff --git a/nyx_mode/LIBNYX_VERSION b/nyx_mode/LIBNYX_VERSION index ed88ec10..9aae19be 100644 --- a/nyx_mode/LIBNYX_VERSION +++ b/nyx_mode/LIBNYX_VERSION @@ -1 +1 @@ -c8a72dc +6833d23 diff --git a/nyx_mode/PACKER_VERSION b/nyx_mode/PACKER_VERSION index 7db88233..cc20a3b6 100644 --- a/nyx_mode/PACKER_VERSION +++ b/nyx_mode/PACKER_VERSION @@ -1 +1 @@ -202bace +bcf3e24 diff --git a/nyx_mode/QEMU-Nyx b/nyx_mode/QEMU-Nyx -Subproject 60c216bc9e4c79834716d4099993d8397a3a8fd +Subproject e5e1c4c21ff9c4dc80e6409d4eab47146c6024c diff --git a/nyx_mode/QEMU_NYX_VERSION b/nyx_mode/QEMU_NYX_VERSION index 98cb134f..c6ed0c6a 100644 --- a/nyx_mode/QEMU_NYX_VERSION +++ b/nyx_mode/QEMU_NYX_VERSION @@ -1 +1 @@ -60c216bc9e +e5e1c4c21ff9c4dc80e6409d4eab47146c6024cd diff --git a/nyx_mode/README.md b/nyx_mode/README.md index eee7d363..7a2a8e6c 100644 --- a/nyx_mode/README.md +++ b/nyx_mode/README.md @@ -84,9 +84,17 @@ Then the final step: we generate the Nyx package configuration: python3 nyx_mode/packer/packer/nyx_config_gen.py PACKAGE-DIRECTORY Kernel ``` - ## Fuzzing with Nyx mode +Note that you need to load the kvm kernel modules for Nyx: +``` +sudo modprobe -r kvm-intel +sudo modprobe -r kvm +sudo modprobe kvm enable_vmware_backdoor=y +sudo modprobe kvm-intel +cat /sys/module/kvm/parameters/enable_vmware_backdoor | grep -q Y && echi OK || echo KVM module problem +``` + All the hard parts are done, fuzzing with Nyx mode is easy - just supply the `PACKAGE-DIRECTORY` as fuzzing target and specify the `-X` option to afl-fuzz: @@ -94,16 +102,8 @@ All the hard parts are done, fuzzing with Nyx mode is easy - just supply the afl-fuzz -i in -o out -X -- ./PACKAGE-DIRECTORY ``` -Most likely your first run will fail because the Linux modules have to be -specially set up, but afl-fuzz will tell you this on startup and how to rectify -the situation: - -``` -sudo modprobe -r kvm-intel # or kvm-amd for AMD processors -sudo modprobe -r kvm -sudo modprobe kvm enable_vmware_backdoor=y -sudo modprobe kvm-intel # or kvm-amd for AMD processors -``` +If you get a forkserver error upon starting then you did not load the Linux +kvm kernel modules, see above. If you want to fuzz in parallel (and you should!), then this has to be done in a special way: @@ -150,12 +150,12 @@ afl-cmin -i in_dir -o out_dir -X -- ./PACKAGE-DIRECTORY On each program startup of one the AFL++ tools in Nyx mode, a Nyx VM is spawned, and a bootstrapping procedure is performed inside the VM to prepare the target environment. As a consequence, due to the bootstrapping procedure, the launch performance is much slower compared to other modes. However, this can be optimized by reusing an existing fuzzing snapshot to avoid the slow re-execution of the bootstrap procedure. -A fuzzing snapshot is automatically created and stored in the output directory at `out_dir/workdir/snapshot/` by the first parent process of `afl-fuzz` if parallel mode is used. To enable this feature, set the path to an existing snapshot directory in the `NYX_REUSE_SNAPSHOT` environment variable and use the tools as usual: +A fuzzing snapshot is automatically created and stored in the output directory at `out_dir/workdir/snapshot/` by the first parent process of `afl-fuzz` if parallel mode is used. To enable this feature, set the path to an existing snapshot directory in the `AFL_NYX_REUSE_SNAPSHOT` environment variable and use the tools as usual: ```shell afl-fuzz -i ./in_dir -o ./out_dir -Y -M 0 ./PACKAGE-DIRECTORY -NYX_REUSE_SNAPSHOT=./out_dir/workdir/snapshot/ afl-analyze -i in_file -X -- ./PACKAGE-DIRECTORY +AFL_NYX_REUSE_SNAPSHOT=./out_dir/workdir/snapshot/ afl-analyze -i in_file -X -- ./PACKAGE-DIRECTORY ``` @@ -311,7 +311,28 @@ command: ``` If you want to disable fast snapshots (except for crashes), you can simply set -the `NYX_DISABLE_SNAPSHOT_MODE` environment variable. +the `AFL_NYX_DISABLE_SNAPSHOT_MODE` environment variable. + +### Nyx crash reports + +If the Nyx agent detects a crash in the target application, it can pass +additional information on that crash to AFL++ (assuming that the agent +implements this feature). For each saved crashing input AFL++ will also create +an additional file in the `crashes` directory with a `.log` file extension. +Crash reports generated by the default agent shipped with the Nyx packer will +contain information such as the faulting address and signal number. +Additionally, if the target is compiled with AddressSanitizer, the crash report +will also contain the entire ASan report. + +From a technical perspective, the crash report is passed from QEMU-Nyx to AFL++ +via a shared memory region called Nyx Auxiliary Buffer which is by default 4096 +bytes in size. In this shared memory region a specific amount is reserved for +the header (1408 bytes) and the remaining bytes can be used to transfer crash +reports (also the `hprintf` feature utilizes the very same shared memory for +transferring data). By default a crash report will be truncated to 2688 bytes. +However, if you want to increase the size of the shared memory region, you can +set the `AFL_NYX_AUX_SIZE` environment variable to a higher value (keep in +mind that this value must be a multiple of 4096). ### Run AFL++Nyx with a custom agent diff --git a/nyx_mode/build_nyx_support.sh b/nyx_mode/build_nyx_support.sh index 581a8292..fda4ec12 100755 --- a/nyx_mode/build_nyx_support.sh +++ b/nyx_mode/build_nyx_support.sh @@ -9,6 +9,21 @@ echo echo "[*] Performing basic sanity checks..." +if [ "$CI" = "true" ]; then + + echo "[-] Error: nyx_mode cannot be tested in the Github CI, skipping ..." + exit 0 + +fi + + +if [ -n "$NO_NYX" ]; then + + echo "[-] Error: the NO_NYX environment variable is set, please unset." + exit 0 + +fi + if [ ! "$(uname -s)" = "Linux" ]; then echo "[-] Error: Nyx mode is only available on Linux." @@ -23,11 +38,17 @@ if [ ! "$(uname -m)" = "x86_64" ]; then fi +cargo help > /dev/null 2>&1 || { + echo "[-] Error: Rust is not installed." + exit 0 +} + echo "[*] Making sure all Nyx is checked out" if git status 1>/dev/null 2>&1; then + set +e git submodule init echo "[*] initializing QEMU-Nyx submodule" git submodule update ./QEMU-Nyx 2>/dev/null # ignore errors @@ -35,6 +56,7 @@ if git status 1>/dev/null 2>&1; then git submodule update ./packer 2>/dev/null # ignore errors echo "[*] initializing libnyx submodule" git submodule update ./libnyx 2>/dev/null # ignore errors + set -e else @@ -48,20 +70,57 @@ test -e packer/.git || { echo "[-] packer not checked out, please install git or test -e libnyx/.git || { echo "[-] libnyx not checked out, please install git or check your internet connection." ; exit 1 ; } test -e QEMU-Nyx/.git || { echo "[-] QEMU-Nyx not checked out, please install git or check your internet connection." ; exit 1 ; } -echo "[*] checking packer init.cpio.gz ..." -if [ ! -f "packer/linux_initramfs/init.cpio.gz" ]; then - (cd packer/linux_initramfs/ && sh pack.sh) + +QEMU_NYX_VERSION="$(cat ./QEMU_NYX_VERSION)" +cd "./QEMU-Nyx" || exit 1 +if [ -n "$NO_CHECKOUT" ]; then + echo "[*] Skipping checkout to $QEMU_NYX_VERSION" +else + echo "[*] Checking out $QEMU_NYX_VERSION" + set +e + sh -c 'git stash' 1>/dev/null 2>/dev/null + git pull 1>/dev/null 2>/dev/null + git checkout "$QEMU_NYX_VERSION" || echo Warning: could not check out to commit $QEMU_NYX_VERSION + set -e fi +cd - > /dev/null -echo "[*] Checking libnyx ..." -if [ ! -f "libnyx/libnyx/target/release/liblibnyx.a" ]; then - (cd libnyx/libnyx && cargo build --release) +PACKER_VERSION="$(cat ./PACKER_VERSION)" +cd "./packer" || exit 1 +if [ -n "$NO_CHECKOUT" ]; then + echo "[*] Skipping checkout to $PACKER_VERSION" +else + echo "[*] Checking out $PACKER_VERSION" + set +e + sh -c 'git stash' 1>/dev/null 2>/dev/null + git pull 1>/dev/null 2>/dev/null + git checkout "$PACKER_VERSION" || echo Warning: could not check out to commit $PACKER_VERSION + set -e fi +cd - > /dev/null -echo "[*] Checking QEMU-Nyx ..." -if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then - (cd QEMU-Nyx && ./compile_qemu_nyx.sh static) +LIBNYX_VERSION="$(cat ./LIBNYX_VERSION)" +cd "./libnyx/" || exit 1 +if [ -n "$NO_CHECKOUT" ]; then + echo "[*] Skipping checkout to $LIBNYX_VERSION" +else + echo "[*] Checking out $LIBNYX_VERSION" + set +e + sh -c 'git stash' 1>/dev/null 2>/dev/null + git pull 1>/dev/null 2>/dev/null + git checkout "$LIBNYX_VERSION" || echo Warning: could not check out to commit $LIBNYX_VERSION + set -e fi +cd - > /dev/null + +echo "[*] checking packer init.cpio.gz ..." +(cd packer/linux_initramfs/ && sh pack.sh) + +echo "[*] Checking libnyx ..." +(cd libnyx/libnyx && cargo build --release) + +echo "[*] Checking QEMU-Nyx ..." +(cd QEMU-Nyx && ./compile_qemu_nyx.sh static ) echo "[*] Checking libnyx.so ..." cp libnyx/libnyx/target/release/liblibnyx.so ../libnyx.so diff --git a/nyx_mode/libnyx b/nyx_mode/libnyx -Subproject 2da7f08b6e0267ccfe64e1320b24cdb29223459 +Subproject 6833d236dfe785a8a23d8c8d79e74c99fa63500 diff --git a/nyx_mode/packer b/nyx_mode/packer -Subproject 202bace888d237e4e8f4507d0eba6791a811554 +Subproject bcf3e248b660764f48af54232a3388389a2dfc2 diff --git a/nyx_mode/update_ref.sh b/nyx_mode/update_ref.sh index 898a803f..146a1255 100755 --- a/nyx_mode/update_ref.sh +++ b/nyx_mode/update_ref.sh @@ -41,7 +41,7 @@ cd .. rm "$UC_VERSION_FILE" echo "$NEW_VERSION" > "$UC_VERSION_FILE" -echo "Done. New XXX version is $NEW_VERSION." +echo "Done. New libnyx version is $NEW_VERSION." UC_VERSION_FILE='./PACKER_VERSION' @@ -68,7 +68,7 @@ cd .. rm "$UC_VERSION_FILE" echo "$NEW_VERSION" > "$UC_VERSION_FILE" -echo "Done. New XXX version is $NEW_VERSION." +echo "Done. New packer version is $NEW_VERSION." UC_VERSION_FILE='./QEMU_NYX_VERSION' @@ -95,5 +95,5 @@ cd .. rm "$UC_VERSION_FILE" echo "$NEW_VERSION" > "$UC_VERSION_FILE" -echo "Done. New XXX version is $NEW_VERSION." +echo "Done. New QEMU-Nyx version is $NEW_VERSION." diff --git a/qemu_mode/QEMUAFL_VERSION b/qemu_mode/QEMUAFL_VERSION index 44ea5345..296745f9 100644 --- a/qemu_mode/QEMUAFL_VERSION +++ b/qemu_mode/QEMUAFL_VERSION @@ -1 +1 @@ -a1321713c7 +a6f0632a65 diff --git a/qemu_mode/README.md b/qemu_mode/README.md index 92038737..b78eb297 100644 --- a/qemu_mode/README.md +++ b/qemu_mode/README.md @@ -193,12 +193,39 @@ Comparative measurements of execution speed or instrumentation coverage will be fairly meaningless if the optimization levels or instrumentation scopes don't match. -## 12) Other features +## 12) Coverage information + +Coverage information about a run of a target binary can be obtained using a +dedicated QEMU user mode plugin enabled at runtime: the `drcov.c` plugin +collects coverage information from the target binary and writes it in the Drcov +format. This file can then be loaded using tools such as +[lighthouse](https://github.com/gaasedelen/lighthouse), +[lightkeeper](https://github.com/WorksButNotTested/lightkeeper) or +[Cartographer](https://github.com/nccgroup/Cartographer). + +To compile the QEMU TCG plugins, run the following command from the `qemuafl` +directory: + +``` +make plugins +``` + +Plugins can be loaded using either the `QEMU_PLUGIN` environment variable or +using the `-plugin` option. For example: + +``` +afl-qemu-trace -plugin qemuafl/build/contrib/plugins/libdrcov.so,arg=filename=/tmp/target.drcov.trace <target> <args> +``` + +This would execute the target binary with the provided arguments and, once done, +would write coverage information at `/tmp/target.drcov.trace`. + +## 13) Other features With `AFL_QEMU_FORCE_DFL`, you force QEMU to ignore the registered signal handlers of the target. -## 13) Gotchas, feedback, bugs +## 14) Gotchas, feedback, bugs If you need to fix up checksums or do other cleanups on mutated test cases, see `afl_custom_post_process` in custom_mutators/examples/example.c for a viable @@ -217,7 +244,7 @@ program may be utilizing. In particular, it does not appear to have full support for AVX2/FMA3. Using binaries for older CPUs or recompiling them with `-march=core2`, can help. -## 14) Alternatives: static rewriting +## 15) Alternatives: static rewriting Statically rewriting binaries just once, instead of attempting to translate them at run time, can be a faster alternative. That said, static rewriting is fraught diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index f59cba78..19336114 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -13,7 +13,7 @@ # counters by Andrea Fioraldi <andreafioraldi@gmail.com> # # Copyright 2015, 2016, 2017 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -132,7 +132,10 @@ echo "Building for CPU target $CPU_TARGET" # --enable-pie seems to give a couple of exec's a second performance # improvement, much to my surprise. Not sure how universal this is.. +# --enable-plugins allows loading TCG plugins at runtime, for example to obtain +# coverage information, and does not seem to negatively impact performance QEMU_CONF_FLAGS=" \ + --enable-plugins \ --audio-drv-list= \ --disable-blobs \ --disable-bochs \ @@ -162,7 +165,6 @@ QEMU_CONF_FLAGS=" \ --disable-numa \ --disable-opengl \ --disable-parallels \ - --disable-plugins \ --disable-qcow1 \ --disable-qed \ --disable-rbd \ @@ -200,6 +202,8 @@ QEMU_CONF_FLAGS=" \ --disable-xfsctl \ --target-list="${CPU_TARGET}-linux-user" \ --without-default-devices \ + --extra-cflags=-Wno-int-conversion \ + --disable-werror \ " if [ -n "${CROSS_PREFIX}" ]; then @@ -213,8 +217,10 @@ if [ "$STATIC" = "1" ]; then echo Building STATIC binary # static PIE causes https://github.com/AFLplusplus/AFLplusplus/issues/892 + # plugin support requires dynamic linking QEMU_CONF_FLAGS="$QEMU_CONF_FLAGS \ --static --disable-pie \ + --disable-plugins \ --extra-cflags=-DAFL_QEMU_STATIC_BUILD=1 \ " @@ -239,7 +245,6 @@ if [ "$DEBUG" = "1" ]; then --enable-debug-stack-usage \ --enable-debug-tcg \ --enable-qom-cast-debug \ - --enable-werror \ " else @@ -250,7 +255,6 @@ else --disable-debug-tcg \ --disable-qom-cast-debug \ --disable-stack-protector \ - --disable-werror \ --disable-docs \ " diff --git a/qemu_mode/fastexit/Makefile b/qemu_mode/fastexit/Makefile index c7b79277..be80207d 100644 --- a/qemu_mode/fastexit/Makefile +++ b/qemu_mode/fastexit/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi <andreafioraldi@gmail.com> # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/Makefile b/qemu_mode/libcompcov/Makefile index 7260df87..4761ac02 100644 --- a/qemu_mode/libcompcov/Makefile +++ b/qemu_mode/libcompcov/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi <andreafioraldi@gmail.com> # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/compcovtest.cc b/qemu_mode/libcompcov/compcovtest.cc index 23215013..11797091 100644 --- a/qemu_mode/libcompcov/compcovtest.cc +++ b/qemu_mode/libcompcov/compcovtest.cc @@ -2,7 +2,7 @@ // // Author: Mateusz Jurczyk (mjurczyk@google.com) // -// Copyright 2019-2023 Google LLC +// Copyright 2019-2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/libcompcov.so.c b/qemu_mode/libcompcov/libcompcov.so.c index b6ee0019..36f7b2e2 100644 --- a/qemu_mode/libcompcov/libcompcov.so.c +++ b/qemu_mode/libcompcov/libcompcov.so.c @@ -5,7 +5,7 @@ Written and maintained by Andrea Fioraldi <andreafioraldi@gmail.com> - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -68,7 +68,11 @@ static int debug_fd = -1; #define MAX_MAPPINGS 1024 -static struct mapping { void *st, *en; } __compcov_ro[MAX_MAPPINGS]; +static struct mapping { + + void *st, *en; + +} __compcov_ro[MAX_MAPPINGS]; static u32 __compcov_ro_cnt; diff --git a/qemu_mode/libqasan/Makefile b/qemu_mode/libqasan/Makefile index 61782894..7366d6f6 100644 --- a/qemu_mode/libqasan/Makefile +++ b/qemu_mode/libqasan/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi <andreafioraldi@gmail.com> # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libqasan/dlmalloc.c b/qemu_mode/libqasan/dlmalloc.c index 5d0b65ce..1919ae26 100644 --- a/qemu_mode/libqasan/dlmalloc.c +++ b/qemu_mode/libqasan/dlmalloc.c @@ -771,8 +771,8 @@ MAX_RELEASE_CHECK_RATE default: 4095 unless not HAVE_MMAP #include "/usr/include/malloc.h" #else /* HAVE_USR_INCLUDE_MALLOC_H */ #ifndef STRUCT_MALLINFO_DECLARED - /* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is - * defined */ + /* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is + * defined */ #define _STRUCT_MALLINFO #define STRUCT_MALLINFO_DECLARED 1 struct mallinfo { @@ -1660,10 +1660,10 @@ extern size_t getpagesize(); #define is_aligned(A) (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0) /* the number of bytes to offset an address to align it */ - #define align_offset(A) \ - ((((size_t)(A)&CHUNK_ALIGN_MASK) == 0) \ - ? 0 \ - : ((MALLOC_ALIGNMENT - ((size_t)(A)&CHUNK_ALIGN_MASK)) & \ + #define align_offset(A) \ + ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0) \ + ? 0 \ + : ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & \ CHUNK_ALIGN_MASK)) /* -------------------------- MMAP preliminaries ------------------------- */ @@ -1715,10 +1715,10 @@ static FORCEINLINE int unixmunmap(void *ptr, size_t size) { #define MUNMAP_DEFAULT(a, s) unixmunmap((a), (s)) #else /* MAP_ANONYMOUS */ - /* - Nearly all versions of mmap support MAP_ANONYMOUS, so the following - is unlikely to be needed, but is supplied just in case. - */ + /* + Nearly all versions of mmap support MAP_ANONYMOUS, so the following + is unlikely to be needed, but is supplied just in case. + */ #define MMAP_FLAGS (MAP_PRIVATE) static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */ #define MMAP_DEFAULT(s) \ @@ -1762,7 +1762,7 @@ static FORCEINLINE void *win32direct_mmap(size_t size) { static FORCEINLINE int win32munmap(void *ptr, size_t size) { MEMORY_BASIC_INFORMATION minfo; - char *cptr = (char *)ptr; + char *cptr = (char *)ptr; while (size) { @@ -1965,7 +1965,7 @@ static FORCEINLINE void x86_clear_lock(int *sl) { #endif /* ... gcc spins locks ... */ - /* How to yield for a spin lock */ + /* How to yield for a spin lock */ #define SPINS_PER_YIELD 63 #if defined(_MSC_VER) #define SLEEP_EX_DURATION 50 /* delay for yield/sleep */ @@ -2008,11 +2008,11 @@ static MLOCK_T malloc_global_mutex = 0; #define CURRENT_THREAD GetCurrentThreadId() #define EQ_OWNER(X, Y) ((X) == (Y)) #else - /* - Note: the following assume that pthread_t is a type that can be - initialized to (casted) zero. If this is not the case, you will need - to somehow redefine these or not use spin locks. - */ + /* + Note: the following assume that pthread_t is a type that can be + initialized to (casted) zero. If this is not the case, you will need + to somehow redefine these or not use spin locks. + */ #define THREAD_ID_T pthread_t #define CURRENT_THREAD pthread_self() #define EQ_OWNER(X, Y) pthread_equal(X, Y) @@ -2169,7 +2169,7 @@ static int pthread_init_lock(MLOCK_T *lk) { #endif /* ... lock types ... */ - /* Common code for all lock types */ + /* Common code for all lock types */ #define USE_LOCK_BIT (2U) #ifndef ACQUIRE_MALLOC_GLOBAL_LOCK @@ -3077,7 +3077,7 @@ static size_t traverse_and_check(mstate m); /* The size of the smallest chunk held in bin with index i */ #define minsize_for_tree_index(i) \ ((SIZE_T_ONE << (((i) >> 1) + TREEBIN_SHIFT)) | \ - (((size_t)((i)&SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1))) + (((size_t)((i) & SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1))) /* ------------------------ Operations on bin maps ----------------------- */ @@ -3245,7 +3245,7 @@ static size_t traverse_and_check(mstate m); #else /* FOOTERS */ - /* Set foot of inuse chunk to be xor of mstate and seed */ + /* Set foot of inuse chunk to be xor of mstate and seed */ #define mark_inuse_foot(M, p, s) \ (((mchunkptr)((char *)(p) + (s)))->prev_foot = \ ((size_t)(M) ^ mparams.magic)) diff --git a/qemu_mode/libqasan/hooks.c b/qemu_mode/libqasan/hooks.c index a9fd0ce9..cf1b0820 100644 --- a/qemu_mode/libqasan/hooks.c +++ b/qemu_mode/libqasan/hooks.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/libqasan.c b/qemu_mode/libqasan/libqasan.c index 12be7778..45f47d5a 100644 --- a/qemu_mode/libqasan/libqasan.c +++ b/qemu_mode/libqasan/libqasan.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/libqasan.h b/qemu_mode/libqasan/libqasan.h index a430c868..f0844e23 100644 --- a/qemu_mode/libqasan/libqasan.h +++ b/qemu_mode/libqasan/libqasan.h @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/malloc.c b/qemu_mode/libqasan/malloc.c index d2db3856..ae470b56 100644 --- a/qemu_mode/libqasan/malloc.c +++ b/qemu_mode/libqasan/malloc.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without @@ -80,8 +80,8 @@ static unsigned char __tmp_alloc_zone[TMP_ZONE_SIZE]; #else // From dlmalloc.c -void *dlmalloc(size_t); -void dlfree(void *); +void *dlmalloc(size_t); +void dlfree(void *); #define backend_malloc dlmalloc #define backend_free dlfree diff --git a/qemu_mode/libqasan/patch.c b/qemu_mode/libqasan/patch.c index 38e0903b..4ce8c3d8 100644 --- a/qemu_mode/libqasan/patch.c +++ b/qemu_mode/libqasan/patch.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/string.c b/qemu_mode/libqasan/string.c index e17cff4b..cd14d57b 100644 --- a/qemu_mode/libqasan/string.c +++ b/qemu_mode/libqasan/string.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/uninstrument.c b/qemu_mode/libqasan/uninstrument.c index e37a9b46..996f2a74 100644 --- a/qemu_mode/libqasan/uninstrument.c +++ b/qemu_mode/libqasan/uninstrument.c @@ -7,7 +7,7 @@ for some strange reason. */ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl -Subproject a1321713c7502c152dd7527555e0f8a800d5522 +Subproject a6f0632a65e101e680dd72643a6128dd180dff7 diff --git a/qemu_mode/unsigaction/Makefile b/qemu_mode/unsigaction/Makefile index c1a7397f..d5e807d8 100644 --- a/qemu_mode/unsigaction/Makefile +++ b/qemu_mode/unsigaction/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi <andreafioraldi@gmail.com> # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/util/qemu_get_symbol_addr.sh b/qemu_mode/util/qemu_get_symbol_addr.sh new file mode 100755 index 00000000..5e00f1b2 --- /dev/null +++ b/qemu_mode/util/qemu_get_symbol_addr.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2024 AFLplusplus +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +target="$1" +symbol="$2" +base="$3" + +test -z "$target" -o -z "$symbol" -o '!' -x "$target" && { + echo "Syntax: $0 executable function [baseaddress]" + echo + echo Help script to calculate the function address of a binary QEMU will load it to. + echo function is e.g. LLVMFuzzerTestOneInput, afl_qemu_driver_stdin, etc. + echo "baseaddress is tried to be auto-detected, you can use 'AFL_QEMU_DEBUG_MAPS=1 afl-qemu-trace ./executable' to see the maps." + exit 1 +} + +file=$(file $target|sed 's/.*: //') + +arch=$(echo $file|awk -F, '{print$2}'|tr -d ' ') +bits=$(echo $file|sed 's/-bit .*//'|sed 's/.* //') +pie=$(echo $file|grep -wqi pie && echo pie) + +test $(uname -s) = "Darwin" && symbol=_"$symbol" +tmp_addr=$(nm "$target" | grep -i "T $symbol" | awk '{print$1}' | tr a-f A-F) + +test -z "$tmp_addr" && { echo Error: function $symbol not found 1>&2; exit 1; } +test -z "$pie" && { echo 0x$tmp_addr; exit 0; } + +test -z "$base" && { + test "$bits" = 32 -o "$bits" = 64 || { echo "Error: could not identify arch (bits=$bits)" 1>&2 ; exit 1; } + test "$arch" = Intel80386 && base=0x40000000 + test "$arch" = x86-64 && base=0x4000000000 + test "$arch" = ARMaarch64 && base=0x5500000000 + # add more here, e.g. "$arch" = ARM +} + +test -z "$base" && { echo "Error: could not identify base address! bits=$bits arch=$arch" 1>&2 ; exit 1; } + +hex_base=$(echo "$base" | awk '{sub("^0x","");print $0}' | tr a-f A-F ) +echo $tmp_addr | echo "ibase=16;obase=10;$hex_base + $tmp_addr" | bc | tr A-F a-f | awk '{print "0x"$0}' +exit 0 diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 5b122741..d089cd08 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-as.c b/src/afl-as.c index 772e31b3..d4ddb94d 100644 --- a/src/afl-as.c +++ b/src/afl-as.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-cc.c b/src/afl-cc.c index 972ac8cd..c872b2eb 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -5,7 +5,7 @@ Written by Michal Zalewski, Laszlo Szekeres and Marc Heuse Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,6 +17,10 @@ #define AFL_MAIN +#ifndef _GNU_SOURCE + #define _GNU_SOURCE 1 +#endif + #include "common.h" #include "config.h" #include "types.h" @@ -32,7 +36,9 @@ #include <limits.h> #include <assert.h> #include <ctype.h> +#include <fcntl.h> #include <sys/stat.h> +#include <sys/mman.h> #if (LLVM_MAJOR - 0 == 0) #undef LLVM_MAJOR @@ -47,23 +53,22 @@ #define LLVM_MINOR 0 #endif -static u8 *obj_path; /* Path to runtime libraries */ -static u8 **cc_params; /* Parameters passed to the real CC */ -static u32 cc_par_cnt = 1; /* Param count, including argv0 */ -static u8 clang_mode; /* Invoked as afl-clang*? */ -static u8 llvm_fullpath[PATH_MAX]; -static u8 instrument_mode, instrument_opt_mode, ngram_size, ctx_k, lto_mode; -static u8 compiler_mode, plusplus_mode, have_instr_env = 0, need_aflpplib = 0; -static u8 have_gcc, have_llvm, have_gcc_plugin, have_lto, have_instr_list = 0; -static u8 *lto_flag = AFL_CLANG_FLTO, *argvnull; -static u8 debug; -static u8 cwd[4096]; -static u8 cmplog_mode; -u8 use_stdin; /* dummy */ -static int passthrough; -// static u8 *march_opt = CFLAGS_OPT; - -enum { +#ifndef MAX_PARAMS_NUM + #define MAX_PARAMS_NUM 2048 +#endif + +/** Global declarations -----BEGIN----- **/ + +typedef enum { + + PARAM_MISS, // not matched + PARAM_SCAN, // scan only + PARAM_KEEP, // kept as-is + PARAM_DROP, // ignored + +} param_st; + +typedef enum { INSTRUMENT_DEFAULT = 0, INSTRUMENT_CLASSIC = 1, @@ -80,7 +85,20 @@ enum { INSTRUMENT_OPT_CTX_K = 64, INSTRUMENT_OPT_CODECOV = 128, -}; +} instrument_mode_id; + +typedef enum { + + UNSET = 0, + LTO = 1, + LLVM = 2, + GCC_PLUGIN = 3, + GCC = 4, + CLANG = 5 + +} compiler_mode_id; + +static u8 cwd[4096]; char instrument_mode_string[18][18] = { @@ -105,17 +123,6 @@ char instrument_mode_string[18][18] = { }; -enum { - - UNSET = 0, - LTO = 1, - LLVM = 2, - GCC_PLUGIN = 3, - GCC = 4, - CLANG = 5 - -}; - char compiler_mode_string[7][12] = { "AUTOSELECT", "LLVM-LTO", "LLVM", "GCC_PLUGIN", @@ -123,6 +130,18 @@ char compiler_mode_string[7][12] = { }; +u8 *instrument_mode_2str(instrument_mode_id i) { + + return instrument_mode_string[i]; + +} + +u8 *compiler_mode_2str(compiler_mode_id i) { + + return compiler_mode_string[i]; + +} + u8 *getthecwd() { if (getcwd(cwd, sizeof(cwd)) == NULL) { @@ -136,26 +155,237 @@ u8 *getthecwd() { } -/* Try to find a specific runtime we need, returns NULL on fail. */ +typedef struct aflcc_state { + + u8 **cc_params; /* Parameters passed to the real CC */ + u32 cc_par_cnt; /* Param count, including argv0 */ + + u8 *argv0; /* Original argv0 (by strdup) */ + u8 *callname; /* Executable file argv0 indicated */ + + u8 debug; + + u8 compiler_mode, plusplus_mode, lto_mode; + + u8 *lto_flag; + + u8 instrument_mode, instrument_opt_mode, ngram_size, ctx_k; + + u8 cmplog_mode; + + u8 have_instr_env, have_gcc, have_clang, have_llvm, have_gcc_plugin, have_lto, + have_optimized_pcguard, have_instr_list; + + u8 fortify_set, x_set, bit_mode, preprocessor_only, have_unroll, have_o, + have_pic, have_c, shared_linking, partial_linking, non_dash, have_fp, + have_flto, have_hidden, have_fortify, have_fcf, have_staticasan, + have_rust_asanrt, have_asan, have_msan, have_ubsan, have_lsan, have_tsan, + have_cfisan; + + // u8 *march_opt; + u8 need_aflpplib; + int passthrough; + + u8 use_stdin; /* dummy */ + u8 *argvnull; /* dummy */ + +} aflcc_state_t; + +void aflcc_state_init(aflcc_state_t *, u8 *argv0); + +u8 *find_object(aflcc_state_t *, u8 *obj); + +void find_built_deps(aflcc_state_t *); + +/* Insert param into the new argv, raise error if MAX_PARAMS_NUM exceeded. */ +static inline void insert_param(aflcc_state_t *aflcc, u8 *param) { + + if (unlikely(aflcc->cc_par_cnt + 1 >= MAX_PARAMS_NUM)) + FATAL("Too many command line parameters, please increase MAX_PARAMS_NUM."); + + aflcc->cc_params[aflcc->cc_par_cnt++] = param; + +} /* - in find_object() we look here: + Insert a param which contains path to the object file. It uses find_object to + get the path based on the name `obj`, and then uses a sprintf like method to + format it with `fmt`. If `fmt` is NULL, the inserted arg is same as the path. + If `msg` provided, it should be an error msg raised if the path can't be + found. `obj` must not be NULL. +*/ +static inline void insert_object(aflcc_state_t *aflcc, u8 *obj, u8 *fmt, + u8 *msg) { + + u8 *_obj_path = find_object(aflcc, obj); + if (!_obj_path) { + + if (msg) + FATAL("%s", msg); + else + FATAL("Unable to find '%s'", obj); + + } else { + + if (fmt) { + + u8 *_obj_path_fmt = alloc_printf(fmt, _obj_path); + ck_free(_obj_path); + aflcc->cc_params[aflcc->cc_par_cnt++] = _obj_path_fmt; + + } else { + + aflcc->cc_params[aflcc->cc_par_cnt++] = _obj_path; + + } + + } + +} + +/* Insert params into the new argv, make clang load the pass. */ +static inline void load_llvm_pass(aflcc_state_t *aflcc, u8 *pass) { + +#if LLVM_MAJOR >= 11 /* use new pass manager */ + #if LLVM_MAJOR < 16 + insert_param(aflcc, "-fexperimental-new-pass-manager"); + #endif + insert_object(aflcc, pass, "-fpass-plugin=%s", 0); +#else + insert_param(aflcc, "-Xclang"); + insert_param(aflcc, "-load"); + insert_param(aflcc, "-Xclang"); + insert_object(aflcc, pass, 0, 0); +#endif + +} + +static inline void debugf_args(int argc, char **argv) { + + DEBUGF("cd '%s';", getthecwd()); + for (int i = 0; i < argc; i++) + SAYF(" '%s'", argv[i]); + SAYF("\n"); + fflush(stdout); + fflush(stderr); + +} - 1. if obj_path is already set we look there first - 2. then we check the $AFL_PATH environment variable location if set - 3. next we check argv[0] if it has path information and use it +void compiler_mode_by_callname(aflcc_state_t *); +void compiler_mode_by_environ(aflcc_state_t *); +void compiler_mode_by_cmdline(aflcc_state_t *, int argc, char **argv); +void instrument_mode_by_environ(aflcc_state_t *); +void mode_final_checkout(aflcc_state_t *, int argc, char **argv); +void mode_notification(aflcc_state_t *); + +void add_real_argv0(aflcc_state_t *); + +void add_defs_common(aflcc_state_t *); +void add_defs_selective_instr(aflcc_state_t *); +void add_defs_persistent_mode(aflcc_state_t *); +void add_defs_fortify(aflcc_state_t *, u8); +void add_defs_lsan_ctrl(aflcc_state_t *); + +param_st parse_fsanitize(aflcc_state_t *, u8 *, u8); +void add_sanitizers(aflcc_state_t *, char **envp); +void add_optimized_pcguard(aflcc_state_t *); +void add_native_pcguard(aflcc_state_t *); + +void add_assembler(aflcc_state_t *); +void add_gcc_plugin(aflcc_state_t *); + +param_st parse_misc_params(aflcc_state_t *, u8 *, u8); +void add_misc_params(aflcc_state_t *); + +param_st parse_linking_params(aflcc_state_t *, u8 *, u8, u8 *skip_next, + char **argv); + +void add_lto_linker(aflcc_state_t *); +void add_lto_passes(aflcc_state_t *); +void add_runtime(aflcc_state_t *); + +/** Global declarations -----END----- **/ + +/* + Init global state struct. We also extract the callname, + check debug options and if in C++ mode here. +*/ +void aflcc_state_init(aflcc_state_t *aflcc, u8 *argv0) { + + // Default NULL/0 is a good start + memset(aflcc, 0, sizeof(aflcc_state_t)); + + aflcc->cc_params = ck_alloc(MAX_PARAMS_NUM * sizeof(u8 *)); + aflcc->cc_par_cnt = 1; + + aflcc->lto_flag = AFL_CLANG_FLTO; + + // aflcc->march_opt = CFLAGS_OPT; + + /* callname & if C++ mode */ + + aflcc->argv0 = ck_strdup(argv0); + + char *cname = NULL; + + if ((cname = strrchr(aflcc->argv0, '/')) != NULL) { + + cname++; + + } else { + + cname = aflcc->argv0; + + } + + aflcc->callname = cname; + + if (strlen(cname) > 2 && (strncmp(cname + strlen(cname) - 2, "++", 2) == 0 || + strstr(cname, "-g++") != NULL)) { + + aflcc->plusplus_mode = 1; + + } + + /* debug */ + + if (getenv("AFL_DEBUG")) { + + aflcc->debug = 1; + if (strcmp(getenv("AFL_DEBUG"), "0") == 0) unsetenv("AFL_DEBUG"); + + } else if (getenv("AFL_QUIET")) { + + be_quiet = 1; + + } + + if ((getenv("AFL_PASSTHROUGH") || getenv("AFL_NOOPT")) && (!aflcc->debug)) { + + be_quiet = 1; + + } + +} + +/* + Try to find a specific runtime we need, in here: + + 1. firstly we check the $AFL_PATH environment variable location if set + 2. next we check argv[0] if it has path information and use it a) we also check ../lib/afl - 4. if 3. failed we check /proc (only Linux, Android, NetBSD, DragonFly, and + 3. if 2. failed we check /proc (only Linux, Android, NetBSD, DragonFly, and FreeBSD with procfs) a) and check here in ../lib/afl too - 5. we look into the AFL_PATH define (usually /usr/local/lib/afl) - 6. we finally try the current directory + 4. we look into the AFL_PATH define (usually /usr/local/lib/afl) + 5. we finally try the current directory if all these attempts fail - we return NULL and the caller has to decide - what to do. + what to do. Otherwise the path to obj would be allocated and returned. */ +u8 *find_object(aflcc_state_t *aflcc, u8 *obj) { -static u8 *find_object(u8 *obj, u8 *argv0) { + u8 *argv0 = aflcc->argv0; u8 *afl_path = getenv("AFL_PATH"); u8 *slash = NULL, *tmp; @@ -164,14 +394,9 @@ static u8 *find_object(u8 *obj, u8 *argv0) { tmp = alloc_printf("%s/%s", afl_path, obj); - if (debug) DEBUGF("Trying %s\n", tmp); - - if (!access(tmp, R_OK)) { + if (aflcc->debug) DEBUGF("Trying %s\n", tmp); - obj_path = afl_path; - return tmp; - - } + if (!access(tmp, R_OK)) { return tmp; } ck_free(tmp); @@ -190,11 +415,11 @@ static u8 *find_object(u8 *obj, u8 *argv0) { tmp = alloc_printf("%s/%s", dir, obj); - if (debug) DEBUGF("Trying %s\n", tmp); + if (aflcc->debug) DEBUGF("Trying %s\n", tmp); if (!access(tmp, R_OK)) { - obj_path = dir; + ck_free(dir); return tmp; } @@ -202,12 +427,10 @@ static u8 *find_object(u8 *obj, u8 *argv0) { ck_free(tmp); tmp = alloc_printf("%s/../lib/afl/%s", dir, obj); - if (debug) DEBUGF("Trying %s\n", tmp); + if (aflcc->debug) DEBUGF("Trying %s\n", tmp); if (!access(tmp, R_OK)) { - u8 *dir2 = alloc_printf("%s/../lib/afl", dir); - obj_path = dir2; ck_free(dir); return tmp; @@ -247,26 +470,18 @@ static u8 *find_object(u8 *obj, u8 *argv0) { *slash = 0; tmp = alloc_printf("%s/%s", exepath, obj); - if (!access(tmp, R_OK)) { + if (aflcc->debug) DEBUGF("Trying %s\n", tmp); - u8 *dir = alloc_printf("%s", exepath); - obj_path = dir; - return tmp; - - } + if (!access(tmp, R_OK)) { return tmp; } ck_free(tmp); tmp = alloc_printf("%s/../lib/afl/%s", exepath, obj); - if (debug) DEBUGF("Trying %s\n", tmp); - - if (!access(tmp, R_OK)) { + if (aflcc->debug) DEBUGF("Trying %s\n", tmp); - u8 *dir = alloc_printf("%s/../lib/afl/", exepath); - obj_path = dir; - return tmp; + if (!access(tmp, R_OK)) { return tmp; } - } + ck_free(tmp); } @@ -283,432 +498,962 @@ static u8 *find_object(u8 *obj, u8 *argv0) { tmp = alloc_printf("%s/%s", AFL_PATH, obj); - if (debug) DEBUGF("Trying %s\n", tmp); + if (aflcc->debug) DEBUGF("Trying %s\n", tmp); - if (!access(tmp, R_OK)) { + if (!access(tmp, R_OK)) { return tmp; } - obj_path = AFL_PATH; - return tmp; + ck_free(tmp); + tmp = alloc_printf("./%s", obj); - } + if (aflcc->debug) DEBUGF("Trying %s\n", tmp); + + if (!access(tmp, R_OK)) { return tmp; } ck_free(tmp); - tmp = alloc_printf("./%s", obj); + if (aflcc->debug) DEBUGF("Trying ... giving up\n"); - if (debug) DEBUGF("Trying %s\n", tmp); + return NULL; - if (!access(tmp, R_OK)) { +} + +/* + Deduce some info about compiler toolchains in current system, + from the building results of AFL++ +*/ +void find_built_deps(aflcc_state_t *aflcc) { + + char *ptr = NULL; - obj_path = "."; - return tmp; +#if defined(__x86_64__) || defined(__i386__) + if ((ptr = find_object(aflcc, "afl-as")) != NULL) { + + #ifndef __APPLE__ + // on OSX clang masquerades as GCC + aflcc->have_gcc = 1; + #endif + aflcc->have_clang = 1; + ck_free(ptr); } - ck_free(tmp); +#endif - if (debug) DEBUGF("Trying ... giving up\n"); + if ((ptr = find_object(aflcc, "SanitizerCoveragePCGUARD.so")) != NULL) { - return NULL; + aflcc->have_optimized_pcguard = 1; + ck_free(ptr); + + } + +#if (LLVM_MAJOR >= 3) + + if ((ptr = find_object(aflcc, "SanitizerCoverageLTO.so")) != NULL) { + + aflcc->have_lto = 1; + ck_free(ptr); + + } + + if ((ptr = find_object(aflcc, "cmplog-routines-pass.so")) != NULL) { + + aflcc->have_llvm = 1; + ck_free(ptr); + + } + +#endif + +#ifdef __ANDROID__ + aflcc->have_llvm = 1; +#endif + + if ((ptr = find_object(aflcc, "afl-gcc-pass.so")) != NULL) { + + aflcc->have_gcc_plugin = 1; + ck_free(ptr); + + } + +#if !defined(__ANDROID__) && !defined(ANDROID) + ptr = find_object(aflcc, "afl-compiler-rt.o"); + + if (!ptr) { + + FATAL( + "Unable to find 'afl-compiler-rt.o'. Please set the AFL_PATH " + "environment variable."); + + } + + if (aflcc->debug) { DEBUGF("rt=%s\n", ptr); } + + ck_free(ptr); +#endif } -void parse_fsanitize(char *string) { +/** compiler_mode & instrument_mode selecting -----BEGIN----- **/ - char *p, *ptr = string + strlen("-fsanitize="); - char *new = malloc(strlen(string) + 1); - char *tmp = malloc(strlen(ptr)); - u32 count = 0, len, ende = 0; +/* Select compiler_mode by callname, such as "afl-clang-fast", etc. */ +void compiler_mode_by_callname(aflcc_state_t *aflcc) { - if (!new || !tmp) { FATAL("could not acquire memory"); } - strcpy(new, "-fsanitize="); + if (strncmp(aflcc->callname, "afl-clang-fast", 14) == 0) { - do { + /* afl-clang-fast is always created there by makefile + just like afl-clang, burdened with special purposes: + - If llvm-config is not available (i.e. LLVM_MAJOR is 0), + or too old, it falls back to LLVM-NATIVE mode and let + the actual compiler complain if doesn't work. + - Otherwise try default llvm instruments except LTO. + */ +#if (LLVM_MAJOR >= 3) + aflcc->compiler_mode = LLVM; +#else + aflcc->compiler_mode = CLANG; +#endif - p = strchr(ptr, ','); - if (!p) { + } else - p = ptr + strlen(ptr) + 1; - ende = 1; +#if (LLVM_MAJOR >= 3) + + if (strncmp(aflcc->callname, "afl-clang-lto", 13) == 0 || + + strncmp(aflcc->callname, "afl-lto", 7) == 0) { + + aflcc->compiler_mode = LTO; + + } else + +#endif + + if (strncmp(aflcc->callname, "afl-gcc-fast", 12) == 0 || + + strncmp(aflcc->callname, "afl-g++-fast", 12) == 0) { + + aflcc->compiler_mode = GCC_PLUGIN; + + } else if (strncmp(aflcc->callname, "afl-gcc", 7) == 0 || + + strncmp(aflcc->callname, "afl-g++", 7) == 0) { + + aflcc->compiler_mode = GCC; + + } else if (strcmp(aflcc->callname, "afl-clang") == 0 || + + strcmp(aflcc->callname, "afl-clang++") == 0) { + + aflcc->compiler_mode = CLANG; + + } + +} + +/* + Select compiler_mode by env AFL_CC_COMPILER. And passthrough mode can be + regarded as a special compiler_mode, so we check for it here, too. +*/ +void compiler_mode_by_environ(aflcc_state_t *aflcc) { + + if (getenv("AFL_PASSTHROUGH") || getenv("AFL_NOOPT")) { + + aflcc->passthrough = 1; + + } + + char *ptr = getenv("AFL_CC_COMPILER"); + + if (!ptr) { return; } + + if (aflcc->compiler_mode) { + + if (!be_quiet) { + + WARNF( + "\"AFL_CC_COMPILER\" is set but a specific compiler was already " + "selected by command line parameter or symlink, ignoring the " + "environment variable!"); } - len = p - ptr; - if (len) { + } else { - strncpy(tmp, ptr, len); - tmp[len] = 0; - // fprintf(stderr, "Found: %s\n", tmp); - ptr += len + 1; - if (*tmp) { + if (strncasecmp(ptr, "LTO", 3) == 0) { - u32 copy = 1; - if (!strcmp(tmp, "fuzzer")) { + aflcc->compiler_mode = LTO; - need_aflpplib = 1; - copy = 0; + } else if (strncasecmp(ptr, "LLVM", 4) == 0) { - } else if (!strncmp(tmp, "fuzzer", 6)) { + aflcc->compiler_mode = LLVM; - copy = 0; + } else if (strncasecmp(ptr, "GCC_P", 5) == 0 || - } + strncasecmp(ptr, "GCC-P", 5) == 0 || + strncasecmp(ptr, "GCCP", 4) == 0) { - if (copy) { + aflcc->compiler_mode = GCC_PLUGIN; - if (count) { strcat(new, ","); } - strcat(new, tmp); - ++count; + } else if (strcasecmp(ptr, "GCC") == 0) { - } + aflcc->compiler_mode = GCC; + + } else if (strcasecmp(ptr, "CLANG") == 0) { + + aflcc->compiler_mode = CLANG; + + } else + + FATAL("Unknown AFL_CC_COMPILER mode: %s\n", ptr); + + } + +} + +/* + Select compiler_mode by command line options --afl-... + If it can be inferred, instrument_mode would also be set. + This can supersedes previous result based on callname + or AFL_CC_COMPILER. And "--afl_noopt"/"--afl-noopt" will + be overwritten by "-g". +*/ +void compiler_mode_by_cmdline(aflcc_state_t *aflcc, int argc, char **argv) { + + char *ptr = NULL; + + for (int i = 1; i < argc; i++) { + + if (strncmp(argv[i], "--afl", 5) == 0) { + + if (!strcmp(argv[i], "--afl_noopt") || !strcmp(argv[i], "--afl-noopt")) { + + aflcc->passthrough = 1; + argv[i] = "-g"; // we have to overwrite it, -g is always good + continue; } - } else { + if (aflcc->compiler_mode && !be_quiet) { - ptr++; /*fprintf(stderr, "NO!\n"); */ + WARNF( + "--afl-... compiler mode supersedes the AFL_CC_COMPILER and " + "symlink compiler selection!"); - } + } - } while (!ende); + ptr = argv[i]; + ptr += 5; + while (*ptr == '-') + ptr++; - strcpy(string, new); - // fprintf(stderr, "string: %s\n", string); - // fprintf(stderr, "new: %s\n", new); + if (strncasecmp(ptr, "LTO", 3) == 0) { + + aflcc->compiler_mode = LTO; + + } else if (strncasecmp(ptr, "LLVM", 4) == 0) { + + aflcc->compiler_mode = LLVM; + + } else if (strncasecmp(ptr, "PCGUARD", 7) == 0 || + + strncasecmp(ptr, "PC-GUARD", 8) == 0) { + + aflcc->compiler_mode = LLVM; + aflcc->instrument_mode = INSTRUMENT_PCGUARD; + + } else if (strcasecmp(ptr, "INSTRIM") == 0 || + + strcasecmp(ptr, "CFG") == 0) { + + FATAL( + "InsTrim instrumentation was removed. Use a modern LLVM and " + "PCGUARD (default in afl-cc).\n"); + + } else if (strcasecmp(ptr, "AFL") == 0 || + + strcasecmp(ptr, "CLASSIC") == 0) { + + aflcc->compiler_mode = LLVM; + aflcc->instrument_mode = INSTRUMENT_CLASSIC; + + } else if (strcasecmp(ptr, "LLVMNATIVE") == 0 || + + strcasecmp(ptr, "NATIVE") == 0 || + strcasecmp(ptr, "LLVM-NATIVE") == 0) { + + aflcc->compiler_mode = LLVM; + aflcc->instrument_mode = INSTRUMENT_LLVMNATIVE; + + } else if (strncasecmp(ptr, "GCC_P", 5) == 0 || + + strncasecmp(ptr, "GCC-P", 5) == 0 || + strncasecmp(ptr, "GCCP", 4) == 0) { + + aflcc->compiler_mode = GCC_PLUGIN; + + } else if (strcasecmp(ptr, "GCC") == 0) { + + aflcc->compiler_mode = GCC; + + } else if (strncasecmp(ptr, "CLANG", 5) == 0) { + + aflcc->compiler_mode = CLANG; + + } else + + FATAL("Unknown --afl-... compiler mode: %s\n", argv[i]); + + } + + } } -static u8 fortify_set = 0, asan_set = 0, x_set = 0, bit_mode = 0, - shared_linking = 0, preprocessor_only = 0, have_unroll = 0, - have_o = 0, have_pic = 0, have_c = 0, partial_linking = 0, - non_dash = 0; +/* + Select instrument_mode by those envs in old style: + - USE_TRACE_PC, AFL_USE_TRACE_PC, AFL_LLVM_USE_TRACE_PC, AFL_TRACE_PC + - AFL_LLVM_CALLER, AFL_LLVM_CTX, AFL_LLVM_CTX_K + - AFL_LLVM_NGRAM_SIZE +*/ +static void instrument_mode_old_environ(aflcc_state_t *aflcc) { + + if (getenv("AFL_LLVM_INSTRIM") || getenv("INSTRIM") || + getenv("INSTRIM_LIB")) { + + FATAL( + "InsTrim instrumentation was removed. Use a modern LLVM and PCGUARD " + "(default in afl-cc).\n"); + + } + + if (getenv("USE_TRACE_PC") || getenv("AFL_USE_TRACE_PC") || + getenv("AFL_LLVM_USE_TRACE_PC") || getenv("AFL_TRACE_PC")) { + + if (aflcc->instrument_mode == 0) + aflcc->instrument_mode = INSTRUMENT_PCGUARD; + else if (aflcc->instrument_mode != INSTRUMENT_PCGUARD) + FATAL("you cannot set AFL_LLVM_INSTRUMENT and AFL_TRACE_PC together"); + + } + + if (getenv("AFL_LLVM_CTX")) aflcc->instrument_opt_mode |= INSTRUMENT_OPT_CTX; + if (getenv("AFL_LLVM_CALLER") || getenv("AFL_LLVM_LTO_CALLER") || + getenv("AFL_LLVM_LTO_CTX")) + aflcc->instrument_opt_mode |= INSTRUMENT_OPT_CALLER; -static void process_params(u32 argc, char **argv) { + if (getenv("AFL_LLVM_NGRAM_SIZE")) { + + aflcc->instrument_opt_mode |= INSTRUMENT_OPT_NGRAM; + aflcc->ngram_size = atoi(getenv("AFL_LLVM_NGRAM_SIZE")); + if (aflcc->ngram_size < 2 || aflcc->ngram_size > NGRAM_SIZE_MAX) + FATAL( + "NGRAM instrumentation mode must be between 2 and NGRAM_SIZE_MAX " + "(%u)", + NGRAM_SIZE_MAX); - if (cc_par_cnt + argc >= 1024) { FATAL("Too many command line parameters"); } + } + + if (getenv("AFL_LLVM_CTX_K")) { + + aflcc->ctx_k = atoi(getenv("AFL_LLVM_CTX_K")); + if (aflcc->ctx_k < 1 || aflcc->ctx_k > CTX_MAX_K) + FATAL("K-CTX instrumentation mode must be between 1 and CTX_MAX_K (%u)", + CTX_MAX_K); + if (aflcc->ctx_k == 1) { - if (lto_mode && argc > 1) { + setenv("AFL_LLVM_CALLER", "1", 1); + unsetenv("AFL_LLVM_CTX_K"); + aflcc->instrument_opt_mode |= INSTRUMENT_OPT_CALLER; - u32 idx; - for (idx = 1; idx < argc; idx++) { + } else { - if (!strncasecmp(argv[idx], "-fpic", 5)) have_pic = 1; + aflcc->instrument_opt_mode |= INSTRUMENT_OPT_CTX_K; } } - // for (u32 x = 0; x < argc; ++x) fprintf(stderr, "[%u] %s\n", x, argv[x]); +} - /* Process the argument list. */ +/* + Select instrument_mode by env 'AFL_LLVM_INSTRUMENT'. + Previous compiler_mode will be superseded, if required by some + values of instrument_mode. +*/ +static void instrument_mode_new_environ(aflcc_state_t *aflcc) { - u8 skip_next = 0; - while (--argc) { + if (!getenv("AFL_LLVM_INSTRUMENT")) { return; } - u8 *cur = *(++argv); + u8 *ptr2 = strtok(getenv("AFL_LLVM_INSTRUMENT"), ":,;"); - if (skip_next) { + while (ptr2) { - skip_next = 0; - continue; + if (strncasecmp(ptr2, "afl", strlen("afl")) == 0 || + strncasecmp(ptr2, "classic", strlen("classic")) == 0) { + + if (aflcc->instrument_mode == INSTRUMENT_LTO) { + + aflcc->instrument_mode = INSTRUMENT_CLASSIC; + aflcc->lto_mode = 1; + + } else if (!aflcc->instrument_mode || + + aflcc->instrument_mode == INSTRUMENT_AFL) { + + aflcc->instrument_mode = INSTRUMENT_AFL; + + } else { + + FATAL("main instrumentation mode already set with %s", + instrument_mode_2str(aflcc->instrument_mode)); + + } } - if (cur[0] != '-') { non_dash = 1; } - if (!strncmp(cur, "--afl", 5)) continue; + if (strncasecmp(ptr2, "pc-guard", strlen("pc-guard")) == 0 || + strncasecmp(ptr2, "pcguard", strlen("pcguard")) == 0) { - if (lto_mode && !strncmp(cur, "-flto=thin", 10)) { + if (!aflcc->instrument_mode || + aflcc->instrument_mode == INSTRUMENT_PCGUARD) - FATAL( - "afl-clang-lto cannot work with -flto=thin. Switch to -flto=full or " - "use afl-clang-fast!"); + aflcc->instrument_mode = INSTRUMENT_PCGUARD; + + else + FATAL("main instrumentation mode already set with %s", + instrument_mode_2str(aflcc->instrument_mode)); } - if (lto_mode && !strncmp(cur, "-fuse-ld=", 9)) continue; - if (lto_mode && !strncmp(cur, "--ld-path=", 10)) continue; - if (!strncmp(cur, "-fno-unroll", 11)) continue; - if (strstr(cur, "afl-compiler-rt") || strstr(cur, "afl-llvm-rt")) continue; - if (!strcmp(cur, "-Wl,-z,defs") || !strcmp(cur, "-Wl,--no-undefined") || - !strcmp(cur, "--no-undefined")) { + if (strncasecmp(ptr2, "llvmnative", strlen("llvmnative")) == 0 || + strncasecmp(ptr2, "llvm-native", strlen("llvm-native")) == 0 || + strncasecmp(ptr2, "native", strlen("native")) == 0) { - continue; + if (!aflcc->instrument_mode || + aflcc->instrument_mode == INSTRUMENT_LLVMNATIVE) + + aflcc->instrument_mode = INSTRUMENT_LLVMNATIVE; + + else + FATAL("main instrumentation mode already set with %s", + instrument_mode_2str(aflcc->instrument_mode)); } - if (compiler_mode == GCC_PLUGIN && !strcmp(cur, "-pipe")) { continue; } + if (strncasecmp(ptr2, "llvmcodecov", strlen("llvmcodecov")) == 0 || + strncasecmp(ptr2, "llvm-codecov", strlen("llvm-codecov")) == 0) { - if (!strcmp(cur, "-z") || !strcmp(cur, "-Wl,-z")) { + if (!aflcc->instrument_mode || + aflcc->instrument_mode == INSTRUMENT_LLVMNATIVE) { - u8 *param = *(argv + 1); - if (!strcmp(param, "defs") || !strcmp(param, "-Wl,defs")) { + aflcc->instrument_mode = INSTRUMENT_LLVMNATIVE; + aflcc->instrument_opt_mode |= INSTRUMENT_OPT_CODECOV; - skip_next = 1; - continue; + } else { + + FATAL("main instrumentation mode already set with %s", + instrument_mode_2str(aflcc->instrument_mode)); } } - if ((compiler_mode == GCC || compiler_mode == GCC_PLUGIN) && - !strncmp(cur, "-stdlib=", 8)) { + if (strncasecmp(ptr2, "cfg", strlen("cfg")) == 0 || + strncasecmp(ptr2, "instrim", strlen("instrim")) == 0) { - if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } - continue; + FATAL( + "InsTrim instrumentation was removed. Use a modern LLVM and " + "PCGUARD (default in afl-cc).\n"); } - if (!strncmp(cur, "-fsanitize-coverage-", 20) && strstr(cur, "list=")) { + if (strncasecmp(ptr2, "lto", strlen("lto")) == 0) { - have_instr_list = 1; + aflcc->lto_mode = 1; + if (!aflcc->instrument_mode || aflcc->instrument_mode == INSTRUMENT_LTO) + + aflcc->instrument_mode = INSTRUMENT_LTO; + + else + FATAL("main instrumentation mode already set with %s", + instrument_mode_2str(aflcc->instrument_mode)); } - if (!strncmp(cur, "-fsanitize=", strlen("-fsanitize=")) && - strchr(cur, ',')) { + if (strcasecmp(ptr2, "gcc") == 0) { - parse_fsanitize(cur); - if (!cur || strlen(cur) <= strlen("-fsanitize=")) { continue; } + if (!aflcc->instrument_mode || aflcc->instrument_mode == INSTRUMENT_GCC) - } else if ((!strncmp(cur, "-fsanitize=fuzzer-", + aflcc->instrument_mode = INSTRUMENT_GCC; - strlen("-fsanitize=fuzzer-")) || - !strncmp(cur, "-fsanitize-coverage", - strlen("-fsanitize-coverage"))) && - (strncmp(cur, "sanitize-coverage-allow", - strlen("sanitize-coverage-allow")) && - strncmp(cur, "sanitize-coverage-deny", - strlen("sanitize-coverage-deny")) && - instrument_mode != INSTRUMENT_LLVMNATIVE)) { + else if (aflcc->instrument_mode != INSTRUMENT_GCC) + FATAL("main instrumentation mode already set with %s", + instrument_mode_2str(aflcc->instrument_mode)); - if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } - continue; + aflcc->compiler_mode = GCC; } - if (need_aflpplib || !strcmp(cur, "-fsanitize=fuzzer")) { + if (strcasecmp(ptr2, "clang") == 0) { - u8 *afllib = find_object("libAFLDriver.a", argv[0]); + if (!aflcc->instrument_mode || aflcc->instrument_mode == INSTRUMENT_CLANG) - if (!be_quiet) { + aflcc->instrument_mode = INSTRUMENT_CLANG; - OKF("Found '-fsanitize=fuzzer', replacing with libAFLDriver.a"); + else if (aflcc->instrument_mode != INSTRUMENT_CLANG) + FATAL("main instrumentation mode already set with %s", + instrument_mode_2str(aflcc->instrument_mode)); - } + aflcc->compiler_mode = CLANG; - if (!afllib) { + } - if (!be_quiet) { + if (strncasecmp(ptr2, "ctx-", strlen("ctx-")) == 0 || + strncasecmp(ptr2, "kctx-", strlen("c-ctx-")) == 0 || + strncasecmp(ptr2, "k-ctx-", strlen("k-ctx-")) == 0) { - WARNF( - "Cannot find 'libAFLDriver.a' to replace '-fsanitize=fuzzer' in " - "the flags - this will fail!"); + u8 *ptr3 = ptr2; + while (*ptr3 && (*ptr3 < '0' || *ptr3 > '9')) + ptr3++; - } + if (!*ptr3) { + + if ((ptr3 = getenv("AFL_LLVM_CTX_K")) == NULL) + FATAL( + "you must set the K-CTX K with (e.g. for value 2) " + "AFL_LLVM_INSTRUMENT=ctx-2"); + + } + + aflcc->ctx_k = atoi(ptr3); + if (aflcc->ctx_k < 1 || aflcc->ctx_k > CTX_MAX_K) + FATAL( + "K-CTX instrumentation option must be between 1 and CTX_MAX_K " + "(%u)", + CTX_MAX_K); + + if (aflcc->ctx_k == 1) { + + aflcc->instrument_opt_mode |= INSTRUMENT_OPT_CALLER; + setenv("AFL_LLVM_CALLER", "1", 1); + unsetenv("AFL_LLVM_CTX_K"); } else { - cc_params[cc_par_cnt++] = afllib; + aflcc->instrument_opt_mode |= (INSTRUMENT_OPT_CTX_K); + u8 *ptr4 = alloc_printf("%u", aflcc->ctx_k); + setenv("AFL_LLVM_CTX_K", ptr4, 1); -#ifdef __APPLE__ - cc_params[cc_par_cnt++] = "-undefined"; - cc_params[cc_par_cnt++] = "dynamic_lookup"; -#endif + } + + } + + if (strcasecmp(ptr2, "ctx") == 0) { + + aflcc->instrument_opt_mode |= INSTRUMENT_OPT_CTX; + setenv("AFL_LLVM_CTX", "1", 1); + + } + + if (strncasecmp(ptr2, "caller", strlen("caller")) == 0) { + + aflcc->instrument_opt_mode |= INSTRUMENT_OPT_CALLER; + setenv("AFL_LLVM_CALLER", "1", 1); + + } + + if (strncasecmp(ptr2, "ngram", strlen("ngram")) == 0) { + + u8 *ptr3 = ptr2 + strlen("ngram"); + while (*ptr3 && (*ptr3 < '0' || *ptr3 > '9')) { + + ptr3++; } - if (need_aflpplib) { + if (!*ptr3) { - need_aflpplib = 0; + if ((ptr3 = getenv("AFL_LLVM_NGRAM_SIZE")) == NULL) + FATAL( + "you must set the NGRAM size with (e.g. for value 2) " + "AFL_LLVM_INSTRUMENT=ngram-2"); - } else { + } - continue; + aflcc->ngram_size = atoi(ptr3); + + if (aflcc->ngram_size < 2 || aflcc->ngram_size > NGRAM_SIZE_MAX) { + + FATAL( + "NGRAM instrumentation option must be between 2 and " + "NGRAM_SIZE_MAX (%u)", + NGRAM_SIZE_MAX); } + aflcc->instrument_opt_mode |= (INSTRUMENT_OPT_NGRAM); + u8 *ptr4 = alloc_printf("%u", aflcc->ngram_size); + setenv("AFL_LLVM_NGRAM_SIZE", ptr4, 1); + } - if (!strcmp(cur, "-m32")) bit_mode = 32; - if (!strcmp(cur, "armv7a-linux-androideabi")) bit_mode = 32; - if (!strcmp(cur, "-m64")) bit_mode = 64; + ptr2 = strtok(NULL, ":,;"); - if (!strcmp(cur, "-fsanitize=address") || !strcmp(cur, "-fsanitize=memory")) - asan_set = 1; + } - if (strstr(cur, "FORTIFY_SOURCE")) fortify_set = 1; +} - if (!strcmp(cur, "-x")) x_set = 1; - if (!strcmp(cur, "-E")) preprocessor_only = 1; - if (!strcmp(cur, "-shared")) shared_linking = 1; - if (!strcmp(cur, "-dynamiclib")) shared_linking = 1; - if (!strcmp(cur, "--target=wasm32-wasi")) passthrough = 1; - if (!strcmp(cur, "-Wl,-r")) partial_linking = 1; - if (!strcmp(cur, "-Wl,-i")) partial_linking = 1; - if (!strcmp(cur, "-Wl,--relocatable")) partial_linking = 1; - if (!strcmp(cur, "-r")) partial_linking = 1; - if (!strcmp(cur, "--relocatable")) partial_linking = 1; - if (!strcmp(cur, "-c")) have_c = 1; +/* + Select instrument_mode by envs, the top wrapper. We check + have_instr_env firstly, then call instrument_mode_old_environ + and instrument_mode_new_environ sequentially. +*/ +void instrument_mode_by_environ(aflcc_state_t *aflcc) { - if (!strncmp(cur, "-O", 2)) have_o = 1; - if (!strncmp(cur, "-funroll-loop", 13)) have_unroll = 1; + if (getenv("AFL_LLVM_INSTRUMENT_FILE") || getenv("AFL_LLVM_WHITELIST") || + getenv("AFL_LLVM_ALLOWLIST") || getenv("AFL_LLVM_DENYLIST") || + getenv("AFL_LLVM_BLOCKLIST")) { - if (*cur == '@') { + aflcc->have_instr_env = 1; - // response file support. - // we have two choices - move everything to the command line or - // rewrite the response files to temporary files and delete them - // afterwards. We choose the first for easiness. - // We do *not* support quotes in the rsp files to cope with spaces in - // filenames etc! If you need that then send a patch! - u8 *filename = cur + 1; - if (debug) { DEBUGF("response file=%s\n", filename); } - FILE *f = fopen(filename, "r"); - struct stat st; + } - // Check not found or empty? let the compiler complain if so. - if (!f || fstat(fileno(f), &st) < 0 || st.st_size < 1) { + if (aflcc->have_instr_env && getenv("AFL_DONT_OPTIMIZE") && !be_quiet) { - cc_params[cc_par_cnt++] = cur; - continue; + WARNF( + "AFL_LLVM_ALLOWLIST/DENYLIST and AFL_DONT_OPTIMIZE cannot be combined " + "for file matching, only function matching!"); - } + } - u8 *tmpbuf = malloc(st.st_size + 1), *ptr; - char **args = malloc(sizeof(char *) * (st.st_size >> 1)); - int count = 1, cont = 0, cont_act = 0; + instrument_mode_old_environ(aflcc); + instrument_mode_new_environ(aflcc); - while (fgets(tmpbuf, st.st_size, f)) { +} - ptr = tmpbuf; - // no leading whitespace - while (isspace(*ptr)) { +/* + Workaround to ensure CALLER, CTX, K-CTX and NGRAM + instrumentation were used correctly. +*/ +static void instrument_opt_mode_exclude(aflcc_state_t *aflcc) { - ++ptr; - cont_act = 0; + if ((aflcc->instrument_opt_mode & INSTRUMENT_OPT_CTX) && + (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CALLER)) { - } + FATAL("you cannot set CTX and CALLER together"); - // no comments, no empty lines - if (*ptr == '#' || *ptr == '\n' || !*ptr) { continue; } - // remove LF - if (ptr[strlen(ptr) - 1] == '\n') { ptr[strlen(ptr) - 1] = 0; } - // remove CR - if (*ptr && ptr[strlen(ptr) - 1] == '\r') { ptr[strlen(ptr) - 1] = 0; } - // handle \ at end of line - if (*ptr && ptr[strlen(ptr) - 1] == '\\') { + } - cont = 1; - ptr[strlen(ptr) - 1] = 0; + if ((aflcc->instrument_opt_mode & INSTRUMENT_OPT_CTX) && + (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CTX_K)) { - } + FATAL("you cannot set CTX and K-CTX together"); - // remove whitespace at end - while (*ptr && isspace(ptr[strlen(ptr) - 1])) { + } - ptr[strlen(ptr) - 1] = 0; - cont = 0; + if ((aflcc->instrument_opt_mode & INSTRUMENT_OPT_CALLER) && + (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CTX_K)) { - } + FATAL("you cannot set CALLER and K-CTX together"); - if (*ptr) { + } - do { + if (aflcc->instrument_opt_mode && aflcc->compiler_mode != LLVM && + !((aflcc->instrument_opt_mode & INSTRUMENT_OPT_CALLER) && + aflcc->compiler_mode == LTO)) + FATAL("CTX, CALLER and NGRAM can only be used in LLVM mode"); - u8 *value = ptr; - while (*ptr && !isspace(*ptr)) { + if (aflcc->instrument_opt_mode && + aflcc->instrument_opt_mode != INSTRUMENT_OPT_CODECOV && + aflcc->instrument_mode != INSTRUMENT_CLASSIC && + !(aflcc->instrument_opt_mode & INSTRUMENT_OPT_CALLER && + aflcc->compiler_mode == LTO)) + FATAL( + "CALLER, CTX and NGRAM instrumentation options can only be used with " + "the LLVM CLASSIC instrumentation mode."); - ++ptr; +} - } +/* + Last step of compiler_mode & instrument_mode selecting. + We have a few of workarounds here, to check any corner cases, + prepare for a series of fallbacks, and raise warnings or errors. +*/ +void mode_final_checkout(aflcc_state_t *aflcc, int argc, char **argv) { - while (*ptr && isspace(*ptr)) { + if (aflcc->instrument_opt_mode && + aflcc->instrument_mode == INSTRUMENT_DEFAULT && + (aflcc->compiler_mode == LLVM || aflcc->compiler_mode == UNSET)) { - *ptr++ = 0; + aflcc->instrument_mode = INSTRUMENT_CLASSIC; + aflcc->compiler_mode = LLVM; - } + } - if (cont_act) { + if (!aflcc->compiler_mode) { - u32 len = strlen(args[count - 1]) + strlen(value) + 1; - u8 *tmp = malloc(len); - snprintf(tmp, len, "%s%s", args[count - 1], value); - free(args[count - 1]); - args[count - 1] = tmp; - cont_act = 0; + // lto is not a default because outside of afl-cc RANLIB and AR have to + // be set to LLVM versions so this would work + if (aflcc->have_llvm) + aflcc->compiler_mode = LLVM; + else if (aflcc->have_gcc_plugin) + aflcc->compiler_mode = GCC_PLUGIN; + else if (aflcc->have_gcc) + aflcc->compiler_mode = GCC; + else if (aflcc->have_clang) + aflcc->compiler_mode = CLANG; + else if (aflcc->have_lto) + aflcc->compiler_mode = LTO; + else + FATAL("no compiler mode available"); - } else { + } - args[count++] = strdup(value); + switch (aflcc->compiler_mode) { + + case GCC: + if (!aflcc->have_gcc) FATAL("afl-gcc is not available on your platform!"); + break; + case CLANG: + if (!aflcc->have_clang) + FATAL("afl-clang is not available on your platform!"); + break; + case LLVM: + if (!aflcc->have_llvm) + FATAL( + "LLVM mode is not available, please install LLVM 13+ and recompile " + "AFL++"); + break; + case GCC_PLUGIN: + if (!aflcc->have_gcc_plugin) + FATAL( + "GCC_PLUGIN mode is not available, install gcc plugin support and " + "recompile AFL++"); + break; + case LTO: + if (!aflcc->have_lto) + FATAL( + "LTO mode is not available, please install LLVM 13+ and lld of the " + "same version and recompile AFL++"); + break; + default: + FATAL("no compiler mode available"); - } + } - } while (*ptr); + if (aflcc->compiler_mode == GCC) { aflcc->instrument_mode = INSTRUMENT_GCC; } - } + if (aflcc->compiler_mode == CLANG) { - if (cont) { + /* if our PCGUARD implementation is not available then silently switch to + native LLVM PCGUARD. Or classic asm instrument is explicitly preferred. */ + if (!aflcc->have_optimized_pcguard && + (aflcc->instrument_mode == INSTRUMENT_DEFAULT || + aflcc->instrument_mode == INSTRUMENT_PCGUARD)) { - cont_act = 1; - cont = 0; + aflcc->instrument_mode = INSTRUMENT_LLVMNATIVE; - } + } else { - } + aflcc->instrument_mode = INSTRUMENT_CLANG; + setenv(CLANG_ENV_VAR, "1", 1); // used by afl-as - if (count) { process_params(count, args); } + } - // we cannot free args[] - free(tmpbuf); + } - continue; + if (aflcc->compiler_mode == LTO) { + + if (aflcc->instrument_mode == 0 || + aflcc->instrument_mode == INSTRUMENT_LTO || + aflcc->instrument_mode == INSTRUMENT_CFG || + aflcc->instrument_mode == INSTRUMENT_PCGUARD) { + + aflcc->lto_mode = 1; + aflcc->instrument_mode = INSTRUMENT_PCGUARD; + + } else if (aflcc->instrument_mode == INSTRUMENT_CLASSIC) { + + aflcc->lto_mode = 1; + + } else { + + if (!be_quiet) { + + WARNF("afl-clang-lto called with mode %s, using that mode instead", + instrument_mode_2str(aflcc->instrument_mode)); + + } } - cc_params[cc_par_cnt++] = cur; + } + + if (aflcc->instrument_mode == 0 && aflcc->compiler_mode < GCC_PLUGIN) { + +#if LLVM_MAJOR >= 7 + #if LLVM_MAJOR < 11 && (LLVM_MAJOR < 10 || LLVM_MINOR < 1) + if (aflcc->have_instr_env) { + + aflcc->instrument_mode = INSTRUMENT_AFL; + if (!be_quiet) { + + WARNF( + "Switching to classic instrumentation because " + "AFL_LLVM_ALLOWLIST/DENYLIST does not work with PCGUARD < 10.0.1."); + + } + + } else + + #endif + aflcc->instrument_mode = INSTRUMENT_PCGUARD; + +#else + aflcc->instrument_mode = INSTRUMENT_AFL; +#endif } -} + if (!aflcc->instrument_opt_mode && aflcc->lto_mode && + aflcc->instrument_mode == INSTRUMENT_CFG) { -/* Copy argv to cc_params, making the necessary edits. */ + aflcc->instrument_mode = INSTRUMENT_PCGUARD; -static void edit_params(u32 argc, char **argv, char **envp) { + } - cc_params = ck_alloc(1024 * sizeof(u8 *)); +#ifndef AFL_CLANG_FLTO + if (aflcc->lto_mode) + FATAL( + "instrumentation mode LTO specified but LLVM support not available " + "(requires LLVM 11 or higher)"); +#endif - if (lto_mode) { + if (aflcc->lto_mode) { - if (lto_flag[0] != '-') + if (aflcc->lto_flag[0] != '-') FATAL( "Using afl-clang-lto is not possible because Makefile magic did not " "identify the correct -flto flag"); else - compiler_mode = LTO; + aflcc->compiler_mode = LTO; } - if (plusplus_mode) { + if (getenv("AFL_LLVM_SKIP_NEVERZERO") && getenv("AFL_LLVM_NOT_ZERO")) + FATAL( + "AFL_LLVM_NOT_ZERO and AFL_LLVM_SKIP_NEVERZERO can not be set " + "together"); - u8 *alt_cxx = getenv("AFL_CXX"); +#if LLVM_MAJOR < 11 && (LLVM_MAJOR < 10 || LLVM_MINOR < 1) - if (!alt_cxx) { + if (aflcc->instrument_mode == INSTRUMENT_PCGUARD && aflcc->have_instr_env) { + + FATAL( + "Instrumentation type PCGUARD does not support " + "AFL_LLVM_ALLOWLIST/DENYLIST! Use LLVM 10.0.1+ instead."); - if (compiler_mode >= GCC_PLUGIN) { + } - if (compiler_mode == GCC) { +#endif - alt_cxx = clang_mode ? "clang++" : "g++"; + instrument_opt_mode_exclude(aflcc); - } else if (compiler_mode == CLANG) { + u8 *ptr2; - alt_cxx = "clang++"; + if ((ptr2 = getenv("AFL_LLVM_DICT2FILE")) != NULL && *ptr2 != '/') + FATAL("AFL_LLVM_DICT2FILE must be set to an absolute file path"); - } else { + if (getenv("AFL_LLVM_LAF_ALL")) { - alt_cxx = "g++"; + setenv("AFL_LLVM_LAF_SPLIT_SWITCHES", "1", 1); + setenv("AFL_LLVM_LAF_SPLIT_COMPARES", "1", 1); + setenv("AFL_LLVM_LAF_SPLIT_FLOATS", "1", 1); + setenv("AFL_LLVM_LAF_TRANSFORM_COMPARES", "1", 1); - } + } + + if (getenv("AFL_LLVM_DICT2FILE") && + (getenv("AFL_LLVM_LAF_SPLIT_SWITCHES") || + getenv("AFL_LLVM_LAF_SPLIT_COMPARES") || + getenv("AFL_LLVM_LAF_SPLIT_FLOATS") || + getenv("AFL_LLVM_LAF_TRANSFORM_COMPARES"))) + FATAL("AFL_LLVM_DICT2FILE is incompatible with AFL_LLVM_LAF_*"); + + aflcc->cmplog_mode = getenv("AFL_CMPLOG") || getenv("AFL_LLVM_CMPLOG") || + getenv("AFL_GCC_CMPLOG"); + +} + +/* + Print welcome message on screen, giving brief notes about + compiler_mode and instrument_mode. +*/ +void mode_notification(aflcc_state_t *aflcc) { + + char *ptr2 = alloc_printf(" + NGRAM-%u", aflcc->ngram_size); + char *ptr3 = alloc_printf(" + K-CTX-%u", aflcc->ctx_k); + + char *ptr1 = alloc_printf( + "%s%s%s%s%s", instrument_mode_2str(aflcc->instrument_mode), + (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CTX) ? " + CTX" : "", + (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CALLER) ? " + CALLER" : "", + (aflcc->instrument_opt_mode & INSTRUMENT_OPT_NGRAM) ? ptr2 : "", + (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CTX_K) ? ptr3 : ""); + + ck_free(ptr2); + ck_free(ptr3); + + if ((isatty(2) && !be_quiet) || aflcc->debug) { + + SAYF(cCYA + "afl-cc" VERSION cRST + " by Michal Zalewski, Laszlo Szekeres, Marc Heuse - mode: %s-%s\n", + compiler_mode_2str(aflcc->compiler_mode), ptr1); + + } + + ck_free(ptr1); + + if (!be_quiet && + (aflcc->compiler_mode == GCC || aflcc->compiler_mode == CLANG)) { + + WARNF( + "You are using outdated instrumentation, install LLVM and/or " + "gcc-plugin and use afl-clang-fast/afl-clang-lto/afl-gcc-fast " + "instead!"); + + } + +} + +/* + Set argv[0] required by execvp. It can be + - specified by env AFL_CXX + - g++ or clang++ + - CLANGPP_BIN or LLVM_BINDIR/clang++ + when in C++ mode, or + - specified by env AFL_CC + - gcc or clang + - CLANG_BIN or LLVM_BINDIR/clang + otherwise. +*/ +void add_real_argv0(aflcc_state_t *aflcc) { + + static u8 llvm_fullpath[PATH_MAX]; + + if (aflcc->plusplus_mode) { + + u8 *alt_cxx = getenv("AFL_CXX"); + + if (!alt_cxx) { + + if (aflcc->compiler_mode == GCC || aflcc->compiler_mode == GCC_PLUGIN) { + + alt_cxx = "g++"; + + } else if (aflcc->compiler_mode == CLANG) { + + alt_cxx = "clang++"; } else { @@ -723,7 +1468,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } - cc_params[0] = alt_cxx; + aflcc->cc_params[0] = alt_cxx; } else { @@ -731,21 +1476,13 @@ static void edit_params(u32 argc, char **argv, char **envp) { if (!alt_cc) { - if (compiler_mode >= GCC_PLUGIN) { - - if (compiler_mode == GCC) { + if (aflcc->compiler_mode == GCC || aflcc->compiler_mode == GCC_PLUGIN) { - alt_cc = clang_mode ? "clang" : "gcc"; + alt_cc = "gcc"; - } else if (compiler_mode == CLANG) { + } else if (aflcc->compiler_mode == CLANG) { - alt_cc = "clang"; - - } else { - - alt_cc = "gcc"; - - } + alt_cc = "clang"; } else { @@ -753,1319 +1490,1284 @@ static void edit_params(u32 argc, char **argv, char **envp) { snprintf(llvm_fullpath, sizeof(llvm_fullpath), "%s/clang", LLVM_BINDIR); else - snprintf(llvm_fullpath, sizeof(llvm_fullpath), "%s", CLANG_BIN); + snprintf(llvm_fullpath, sizeof(llvm_fullpath), CLANG_BIN); alt_cc = llvm_fullpath; } } - cc_params[0] = alt_cc; + aflcc->cc_params[0] = alt_cc; } - if (compiler_mode == GCC || compiler_mode == CLANG) { +} - cc_params[cc_par_cnt++] = "-B"; - cc_params[cc_par_cnt++] = obj_path; +/** compiler_mode & instrument_mode selecting -----END----- **/ - if (clang_mode || compiler_mode == CLANG) { +/** Macro defs for the preprocessor -----BEGIN----- **/ - cc_params[cc_par_cnt++] = "-no-integrated-as"; +void add_defs_common(aflcc_state_t *aflcc) { - } + insert_param(aflcc, "-D__AFL_COMPILER=1"); + insert_param(aflcc, "-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"); - } - - if (compiler_mode == GCC_PLUGIN) { +} - char *fplugin_arg; +/* + __afl_coverage macro defs. See + instrumentation/README.instrument_list.md# + 2-selective-instrumentation-with-_afl_coverage-directives +*/ +void add_defs_selective_instr(aflcc_state_t *aflcc) { - if (cmplog_mode) { + if (aflcc->plusplus_mode) { - fplugin_arg = - alloc_printf("-fplugin=%s/afl-gcc-cmplog-pass.so", obj_path); - cc_params[cc_par_cnt++] = fplugin_arg; - fplugin_arg = - alloc_printf("-fplugin=%s/afl-gcc-cmptrs-pass.so", obj_path); - cc_params[cc_par_cnt++] = fplugin_arg; + insert_param(aflcc, + "-D__AFL_COVERAGE()=int __afl_selective_coverage = 1;" + "extern \"C\" void __afl_coverage_discard();" + "extern \"C\" void __afl_coverage_skip();" + "extern \"C\" void __afl_coverage_on();" + "extern \"C\" void __afl_coverage_off();"); - } + } else { - fplugin_arg = alloc_printf("-fplugin=%s/afl-gcc-pass.so", obj_path); - cc_params[cc_par_cnt++] = fplugin_arg; - cc_params[cc_par_cnt++] = "-fno-if-conversion"; - cc_params[cc_par_cnt++] = "-fno-if-conversion2"; + insert_param(aflcc, + "-D__AFL_COVERAGE()=int __afl_selective_coverage = 1;" + "void __afl_coverage_discard();" + "void __afl_coverage_skip();" + "void __afl_coverage_on();" + "void __afl_coverage_off();"); } - if (compiler_mode == LLVM || compiler_mode == LTO) { + insert_param( + aflcc, + "-D__AFL_COVERAGE_START_OFF()=int __afl_selective_coverage_start_off = " + "1;"); + insert_param(aflcc, "-D__AFL_COVERAGE_ON()=__afl_coverage_on()"); + insert_param(aflcc, "-D__AFL_COVERAGE_OFF()=__afl_coverage_off()"); + insert_param(aflcc, "-D__AFL_COVERAGE_DISCARD()=__afl_coverage_discard()"); + insert_param(aflcc, "-D__AFL_COVERAGE_SKIP()=__afl_coverage_skip()"); - cc_params[cc_par_cnt++] = "-Wno-unused-command-line-argument"; +} - if (lto_mode && have_instr_env) { +/* + Macro defs for persistent mode. As documented in + instrumentation/README.persistent_mode.md, deferred forkserver initialization + and persistent mode are not available in afl-gcc and afl-clang. +*/ +void add_defs_persistent_mode(aflcc_state_t *aflcc) { -#if LLVM_MAJOR >= 11 /* use new pass manager */ - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = alloc_printf( - "-fpass-plugin=%s/afl-llvm-lto-instrumentlist.so", obj_path); -#else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/afl-llvm-lto-instrumentlist.so", obj_path); -#endif + if (aflcc->compiler_mode == GCC || aflcc->compiler_mode == CLANG) return; - } + insert_param(aflcc, "-D__AFL_HAVE_MANUAL_CONTROL=1"); - if (getenv("AFL_LLVM_DICT2FILE")) { + /* When the user tries to use persistent or deferred forkserver modes by + appending a single line to the program, we want to reliably inject a + signature into the binary (to be picked up by afl-fuzz) and we want + to call a function from the runtime .o file. This is unnecessarily + painful for three reasons: -#if LLVM_MAJOR >= 11 /* use new pass manager */ - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/afl-llvm-dict2file.so", obj_path); -#else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/afl-llvm-dict2file.so", obj_path); -#endif + 1) We need to convince the compiler not to optimize out the signature. + This is done with __attribute__((used)). - } + 2) We need to convince the linker, when called with -Wl,--gc-sections, + not to do the same. This is done by forcing an assignment to a + 'volatile' pointer. - // laf - if (getenv("LAF_SPLIT_SWITCHES") || getenv("AFL_LLVM_LAF_SPLIT_SWITCHES")) { + 3) We need to declare __afl_persistent_loop() in the global namespace, + but doing this within a method in a class is hard - :: and extern "C" + are forbidden and __attribute__((alias(...))) doesn't work. Hence the + __asm__ aliasing trick. -#if LLVM_MAJOR >= 11 /* use new pass manager */ - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); -#else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/split-switches-pass.so", obj_path); -#endif + */ - } + insert_param(aflcc, + "-D__AFL_FUZZ_INIT()=" + "int __afl_sharedmem_fuzzing = 1;" + "extern __attribute__((visibility(\"default\"))) " + "unsigned int *__afl_fuzz_len;" + "extern __attribute__((visibility(\"default\"))) " + "unsigned char *__afl_fuzz_ptr;" + "unsigned char __afl_fuzz_alt[1048576];" + "unsigned char *__afl_fuzz_alt_ptr = __afl_fuzz_alt;"); - if (getenv("LAF_TRANSFORM_COMPARES") || - getenv("AFL_LLVM_LAF_TRANSFORM_COMPARES")) { + insert_param(aflcc, + "-D__AFL_FUZZ_TESTCASE_BUF=(__afl_fuzz_ptr ? __afl_fuzz_ptr : " + "__afl_fuzz_alt_ptr)"); -#if LLVM_MAJOR >= 11 /* use new pass manager */ - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/compare-transform-pass.so", obj_path); + insert_param( + aflcc, + "-D__AFL_FUZZ_TESTCASE_LEN=(__afl_fuzz_ptr ? *__afl_fuzz_len : " + "(*__afl_fuzz_len = read(0, __afl_fuzz_alt_ptr, 1048576)) == 0xffffffff " + "? 0 : *__afl_fuzz_len)"); + + insert_param( + aflcc, + "-D__AFL_LOOP(_A)=" + "({ static volatile const char *_B __attribute__((used,unused)); " + " _B = (const char*)\"" PERSIST_SIG + "\"; " + "extern __attribute__((visibility(\"default\"))) int __afl_connected;" +#ifdef __APPLE__ + "__attribute__((visibility(\"default\"))) " + "int _L(unsigned int) __asm__(\"___afl_persistent_loop\"); " #else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/compare-transform-pass.so", obj_path); -#endif + "__attribute__((visibility(\"default\"))) " + "int _L(unsigned int) __asm__(\"__afl_persistent_loop\"); " +#endif /* ^__APPLE__ */ + // if afl is connected, we run _A times, else once. + "_L(__afl_connected ? _A : 1); })"); + + insert_param( + aflcc, + "-D__AFL_INIT()=" + "do { static volatile const char *_A __attribute__((used,unused)); " + " _A = (const char*)\"" DEFER_SIG + "\"; " +#ifdef __APPLE__ + "__attribute__((visibility(\"default\"))) " + "void _I(void) __asm__(\"___afl_manual_init\"); " +#else + "__attribute__((visibility(\"default\"))) " + "void _I(void) __asm__(\"__afl_manual_init\"); " +#endif /* ^__APPLE__ */ + "_I(); } while (0)"); - } +} - if (getenv("LAF_SPLIT_COMPARES") || getenv("AFL_LLVM_LAF_SPLIT_COMPARES") || - getenv("AFL_LLVM_LAF_SPLIT_FLOATS")) { +/* + Control macro def of _FORTIFY_SOURCE. It will do nothing + if we detect this routine has been called previously, or + the macro already here in these existing args. +*/ +void add_defs_fortify(aflcc_state_t *aflcc, u8 action) { -#if LLVM_MAJOR >= 11 /* use new pass manager */ - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/split-compares-pass.so", obj_path); -#else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/split-compares-pass.so", obj_path); -#endif + if (aflcc->have_fortify) { return; } - } + switch (action) { - // /laf + case 1: + insert_param(aflcc, "-D_FORTIFY_SOURCE=1"); + break; - unsetenv("AFL_LD"); - unsetenv("AFL_LD_CALLER"); + case 2: + insert_param(aflcc, "-D_FORTIFY_SOURCE=2"); + break; - if (cmplog_mode) { + default: // OFF + insert_param(aflcc, "-U_FORTIFY_SOURCE"); + break; - cc_params[cc_par_cnt++] = "-fno-inline"; + } -#if LLVM_MAJOR >= 11 /* use new pass manager */ - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/cmplog-switches-pass.so", obj_path); - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); -#else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/cmplog-switches-pass.so", obj_path); + aflcc->have_fortify = 1; - // reuse split switches from laf - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/split-switches-pass.so", obj_path); -#endif +} - } +/* Macro defs of __AFL_LEAK_CHECK, __AFL_LSAN_ON and __AFL_LSAN_OFF */ +void add_defs_lsan_ctrl(aflcc_state_t *aflcc) { - //#if LLVM_MAJOR >= 13 - // // Use the old pass manager in LLVM 14 which the AFL++ passes still - // use. cc_params[cc_par_cnt++] = "-flegacy-pass-manager"; - //#endif + insert_param(aflcc, "-includesanitizer/lsan_interface.h"); + insert_param( + aflcc, + "-D__AFL_LEAK_CHECK()={if(__lsan_do_recoverable_leak_check() > 0) " + "_exit(23); }"); + insert_param(aflcc, "-D__AFL_LSAN_OFF()=__lsan_disable();"); + insert_param(aflcc, "-D__AFL_LSAN_ON()=__lsan_enable();"); - if (lto_mode && !have_c) { +} - u8 *ld_path = NULL; - if (getenv("AFL_REAL_LD")) { +/** Macro defs for the preprocessor -----END----- **/ - ld_path = strdup(getenv("AFL_REAL_LD")); +/** About -fsanitize -----BEGIN----- **/ - } else { +/* For input "-fsanitize=...", it: - ld_path = strdup(AFL_REAL_LD); + 1. may have various OOB traps :) if ... doesn't contain ',' or + the input has bad syntax such as "-fsantiz=," + 2. strips any fuzzer* in ... and writes back (may result in "-fsanitize=") + 3. rets 1 if exactly "fuzzer" found, otherwise rets 0 +*/ +static u8 fsanitize_fuzzer_comma(char *string) { - } + u8 detect_single_fuzzer = 0; - if (!ld_path || !*ld_path) { + char *p, *ptr = string + strlen("-fsanitize="); + // ck_alloc will check alloc failure + char *new = ck_alloc(strlen(string) + 1); + char *tmp = ck_alloc(strlen(ptr) + 1); + u32 count = 0, len, ende = 0; - if (ld_path) { + strcpy(new, "-fsanitize="); - // Freeing empty string - free(ld_path); + do { - } + p = strchr(ptr, ','); + if (!p) { - ld_path = strdup("ld.lld"); + p = ptr + strlen(ptr) + 1; + ende = 1; - } + } - if (!ld_path) { PFATAL("Could not allocate mem for ld_path"); } -#if defined(AFL_CLANG_LDPATH) && LLVM_MAJOR >= 12 - cc_params[cc_par_cnt++] = alloc_printf("--ld-path=%s", ld_path); -#else - cc_params[cc_par_cnt++] = alloc_printf("-fuse-ld=%s", ld_path); -#endif - free(ld_path); + len = p - ptr; + if (len) { -#if defined(AFL_CLANG_LDPATH) && LLVM_MAJOR >= 15 - // The NewPM implementation only works fully since LLVM 15. - cc_params[cc_par_cnt++] = alloc_printf( - "-Wl,--load-pass-plugin=%s/SanitizerCoverageLTO.so", obj_path); -#elif defined(AFL_CLANG_LDPATH) && LLVM_MAJOR >= 13 - cc_params[cc_par_cnt++] = "-Wl,--lto-legacy-pass-manager"; - cc_params[cc_par_cnt++] = - alloc_printf("-Wl,-mllvm=-load=%s/SanitizerCoverageLTO.so", obj_path); -#else - cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; - cc_params[cc_par_cnt++] = - alloc_printf("-Wl,-mllvm=-load=%s/SanitizerCoverageLTO.so", obj_path); -#endif + strncpy(tmp, ptr, len); + tmp[len] = 0; + // fprintf(stderr, "Found: %s\n", tmp); + ptr += len + 1; + if (*tmp) { - cc_params[cc_par_cnt++] = "-Wl,--allow-multiple-definition"; - cc_params[cc_par_cnt++] = lto_flag; + u32 copy = 1; + if (!strcmp(tmp, "fuzzer")) { - } else { + detect_single_fuzzer = 1; + copy = 0; - if (instrument_mode == INSTRUMENT_PCGUARD) { + } else if (!strncmp(tmp, "fuzzer", 6)) { -#if LLVM_MAJOR >= 11 - #if defined __ANDROID__ || ANDROID - cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; - instrument_mode = INSTRUMENT_LLVMNATIVE; - #else - if (have_instr_list) { - - if (!be_quiet) - SAYF( - "Using unoptimized trace-pc-guard, due usage of " - "-fsanitize-coverage-allow/denylist, you can use " - "AFL_LLVM_ALLOWLIST/AFL_LLMV_DENYLIST instead.\n"); - cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; - instrument_mode = INSTRUMENT_LLVMNATIVE; - - } else { - - #if LLVM_MAJOR >= 11 /* use new pass manager */ - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = alloc_printf( - "-fpass-plugin=%s/SanitizerCoveragePCGUARD.so", obj_path); - #else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/SanitizerCoveragePCGUARD.so", obj_path); - #endif + copy = 0; } - #endif -#else - #if LLVM_MAJOR >= 4 - if (!be_quiet) - SAYF( - "Using unoptimized trace-pc-guard, upgrade to llvm 10.0.1+ for " - "enhanced version.\n"); - cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; - instrument_mode = INSTRUMENT_LLVMNATIVE; - #else - FATAL("pcguard instrumentation requires llvm 4.0.1+"); - #endif -#endif + if (copy) { - } else if (instrument_mode == INSTRUMENT_LLVMNATIVE) { + if (count) { strcat(new, ","); } + strcat(new, tmp); + ++count; -#if LLVM_MAJOR >= 4 - if (instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { + } - #if LLVM_MAJOR >= 6 - cc_params[cc_par_cnt++] = - "-fsanitize-coverage=trace-pc-guard,bb,no-prune,pc-table"; - #else - FATAL("pcguard instrumentation with pc-table requires llvm 6.0.1+"); - #endif + } - } else { + } else { - cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; + ptr++; - } + } -#else - FATAL("pcguard instrumentation requires llvm 4.0.1+"); -#endif + } while (!ende); - } else { + strcpy(string, new); -#if LLVM_MAJOR >= 11 /* use new pass manager */ - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/afl-llvm-pass.so", obj_path); -#else + ck_free(tmp); + ck_free(new); - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); -#endif + return detect_single_fuzzer; - } +} - } +/* + Parse and process possible -fsanitize related args, return PARAM_MISS + if nothing matched. We have 3 main tasks here for these args: + - Check which one of those sanitizers present here. + - Check if libfuzzer present. We need to block the request of enable + libfuzzer, and link harness with our libAFLDriver.a later. + - Check if SanCov allow/denylist options present. We need to try switching + to LLVMNATIVE instead of using our optimized PCGUARD anyway. If we + can't make it finally for various reasons, just drop these options. +*/ +param_st parse_fsanitize(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan) { + + param_st final_ = PARAM_MISS; + +// MACRO START +#define HAVE_SANITIZER_SCAN_KEEP(v, k) \ + do { \ + \ + if (strstr(cur_argv, "=" STRINGIFY(k)) || \ + strstr(cur_argv, "," STRINGIFY(k))) { \ + \ + if (scan) { \ + \ + aflcc->have_##v = 1; \ + final_ = PARAM_SCAN; \ + \ + } else { \ + \ + final_ = PARAM_KEEP; \ + \ + } \ + \ + } \ + \ + } while (0) + + // MACRO END + + if (!strncmp(cur_argv, "-fsanitize=", strlen("-fsanitize="))) { + + HAVE_SANITIZER_SCAN_KEEP(asan, address); + HAVE_SANITIZER_SCAN_KEEP(msan, memory); + HAVE_SANITIZER_SCAN_KEEP(ubsan, undefined); + HAVE_SANITIZER_SCAN_KEEP(tsan, thread); + HAVE_SANITIZER_SCAN_KEEP(lsan, leak); + HAVE_SANITIZER_SCAN_KEEP(cfisan, cfi); - if (cmplog_mode) { + } -#if LLVM_MAJOR >= 11 - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = alloc_printf( - "-fpass-plugin=%s/cmplog-instructions-pass.so", obj_path); - #if LLVM_MAJOR < 16 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - #endif - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/cmplog-routines-pass.so", obj_path); -#else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/cmplog-instructions-pass.so", obj_path); - - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = - alloc_printf("%s/cmplog-routines-pass.so", obj_path); -#endif +#undef HAVE_SANITIZER_SCAN_KEEP + + // We can't use a "else if" there, because some of the following + // matching rules overlap with those in the if-statement above. + if (!strcmp(cur_argv, "-fsanitize=fuzzer")) { + + if (scan) { + + aflcc->need_aflpplib = 1; + final_ = PARAM_SCAN; + + } else { + + final_ = PARAM_DROP; } - // cc_params[cc_par_cnt++] = "-Qunused-arguments"; + } else if (!strncmp(cur_argv, "-fsanitize=", strlen("-fsanitize=")) && + + strchr(cur_argv, ',') && + !strstr(cur_argv, "=,")) { // avoid OOB errors + + if (scan) { - if (lto_mode && argc > 1) { + u8 *cur_argv_ = ck_strdup(cur_argv); - u32 idx; - for (idx = 1; idx < argc; idx++) { + if (fsanitize_fuzzer_comma(cur_argv_)) { - if (!strncasecmp(argv[idx], "-fpic", 5)) have_pic = 1; + aflcc->need_aflpplib = 1; + final_ = PARAM_SCAN; } - } + ck_free(cur_argv_); - } + } else { - /* Inspect the command line parameters. */ + fsanitize_fuzzer_comma(cur_argv); + if (!cur_argv || strlen(cur_argv) <= strlen("-fsanitize=")) + final_ = PARAM_DROP; // this means it only has "fuzzer" previously. - process_params(argc, argv); + } - if (!have_pic) { cc_params[cc_par_cnt++] = "-fPIC"; } + } else if (!strncmp(cur_argv, "-fsanitize-coverage-", 20) && - // in case LLVM is installed not via a package manager or "make install" - // e.g. compiled download or compiled from github then its ./lib directory - // might not be in the search path. Add it if so. - u8 *libdir = strdup(LLVM_LIBDIR); - if (plusplus_mode && strlen(libdir) && strncmp(libdir, "/usr", 4) && - strncmp(libdir, "/lib", 4)) { + strstr(cur_argv, "list=")) { - cc_params[cc_par_cnt++] = "-Wl,-rpath"; - cc_params[cc_par_cnt++] = libdir; + if (scan) { - } else { + aflcc->have_instr_list = 1; + final_ = PARAM_SCAN; - free(libdir); + } else { - } + if (aflcc->instrument_mode != INSTRUMENT_LLVMNATIVE) { - if (getenv("AFL_HARDEN")) { + if (!be_quiet) { WARNF("Found '%s' - stripping!", cur_argv); } + final_ = PARAM_DROP; + + } else { - cc_params[cc_par_cnt++] = "-fstack-protector-all"; + final_ = PARAM_KEEP; - if (!fortify_set) cc_params[cc_par_cnt++] = "-D_FORTIFY_SOURCE=2"; + } + + } } - if (!asan_set) { + if (final_ == PARAM_KEEP) insert_param(aflcc, cur_argv); - if (getenv("AFL_USE_ASAN")) { + return final_; - if (getenv("AFL_USE_MSAN")) FATAL("ASAN and MSAN are mutually exclusive"); +} - if (getenv("AFL_HARDEN")) - FATAL("ASAN and AFL_HARDEN are mutually exclusive"); +/* + Add params for sanitizers. Here we need to consider: + - Use static runtime for asan, as much as possible. + - ASAN, MSAN, AFL_HARDEN are mutually exclusive. + - Add options if not found there, on request of AFL_USE_ASAN, AFL_USE_MSAN, + etc. + - Update have_* so that functions called after this can have correct context. + However this also means any functions called before should NOT depend on + these have_*, otherwise they may not work as expected. +*/ +void add_sanitizers(aflcc_state_t *aflcc, char **envp) { - cc_params[cc_par_cnt++] = "-U_FORTIFY_SOURCE"; - cc_params[cc_par_cnt++] = "-fsanitize=address"; + if (getenv("AFL_USE_ASAN") || aflcc->have_asan) { - } else if (getenv("AFL_USE_MSAN")) { + if (getenv("AFL_USE_MSAN") || aflcc->have_msan) + FATAL("ASAN and MSAN are mutually exclusive"); - if (getenv("AFL_USE_ASAN")) FATAL("ASAN and MSAN are mutually exclusive"); + if (getenv("AFL_HARDEN")) + FATAL("ASAN and AFL_HARDEN are mutually exclusive"); - if (getenv("AFL_HARDEN")) - FATAL("MSAN and AFL_HARDEN are mutually exclusive"); + if (aflcc->compiler_mode == GCC_PLUGIN && !aflcc->have_staticasan) { - cc_params[cc_par_cnt++] = "-U_FORTIFY_SOURCE"; - cc_params[cc_par_cnt++] = "-fsanitize=memory"; + insert_param(aflcc, "-static-libasan"); } - } + add_defs_fortify(aflcc, 0); + if (!aflcc->have_asan) { - if (getenv("AFL_USE_UBSAN")) { + insert_param(aflcc, "-fsanitize=address"); + insert_param(aflcc, "-fno-common"); - cc_params[cc_par_cnt++] = "-fsanitize=undefined"; - cc_params[cc_par_cnt++] = "-fsanitize-undefined-trap-on-error"; - cc_params[cc_par_cnt++] = "-fno-sanitize-recover=all"; - cc_params[cc_par_cnt++] = "-fno-omit-frame-pointer"; + } - } + aflcc->have_asan = 1; + + } else if (getenv("AFL_USE_MSAN") || aflcc->have_msan) { + + if (getenv("AFL_USE_ASAN") || aflcc->have_asan) + FATAL("ASAN and MSAN are mutually exclusive"); - if (getenv("AFL_USE_TSAN")) { + if (getenv("AFL_HARDEN")) + FATAL("MSAN and AFL_HARDEN are mutually exclusive"); - cc_params[cc_par_cnt++] = "-fsanitize=thread"; - cc_params[cc_par_cnt++] = "-fno-omit-frame-pointer"; + add_defs_fortify(aflcc, 0); + if (!aflcc->have_msan) { insert_param(aflcc, "-fsanitize=memory"); } + aflcc->have_msan = 1; } - if (getenv("AFL_USE_LSAN")) { + if (getenv("AFL_USE_UBSAN") || aflcc->have_ubsan) { - cc_params[cc_par_cnt++] = "-fsanitize=leak"; - cc_params[cc_par_cnt++] = "-includesanitizer/lsan_interface.h"; - cc_params[cc_par_cnt++] = - "-D__AFL_LEAK_CHECK()={if(__lsan_do_recoverable_leak_check() > 0) " - "_exit(23); }"; - cc_params[cc_par_cnt++] = "-D__AFL_LSAN_OFF()=__lsan_disable();"; - cc_params[cc_par_cnt++] = "-D__AFL_LSAN_ON()=__lsan_enable();"; + if (!aflcc->have_ubsan) { - } + insert_param(aflcc, "-fsanitize=undefined"); + insert_param(aflcc, "-fsanitize-undefined-trap-on-error"); + insert_param(aflcc, "-fno-sanitize-recover=all"); - if (getenv("AFL_USE_CFISAN")) { + } - if (compiler_mode == GCC_PLUGIN || compiler_mode == GCC) { + if (!aflcc->have_fp) { - cc_params[cc_par_cnt++] = "-fcf-protection=full"; + insert_param(aflcc, "-fno-omit-frame-pointer"); + aflcc->have_fp = 1; - } else { + } - if (!lto_mode) { + aflcc->have_ubsan = 1; - uint32_t i = 0, found = 0; - while (envp[i] != NULL && !found) - if (strncmp("-flto", envp[i++], 5) == 0) found = 1; - if (!found) cc_params[cc_par_cnt++] = "-flto"; + } - } + if (getenv("AFL_USE_TSAN") || aflcc->have_tsan) { + + if (!aflcc->have_fp) { - cc_params[cc_par_cnt++] = "-fsanitize=cfi"; - cc_params[cc_par_cnt++] = "-fvisibility=hidden"; + insert_param(aflcc, "-fno-omit-frame-pointer"); + aflcc->have_fp = 1; } + if (!aflcc->have_tsan) { insert_param(aflcc, "-fsanitize=thread"); } + aflcc->have_tsan = 1; + } - if (!getenv("AFL_DONT_OPTIMIZE")) { + if (getenv("AFL_USE_LSAN") && !aflcc->have_lsan) { - cc_params[cc_par_cnt++] = "-g"; - if (!have_o) cc_params[cc_par_cnt++] = "-O3"; - if (!have_unroll) cc_params[cc_par_cnt++] = "-funroll-loops"; - // if (strlen(march_opt) > 1 && march_opt[0] == '-') - // cc_params[cc_par_cnt++] = march_opt; + insert_param(aflcc, "-fsanitize=leak"); + add_defs_lsan_ctrl(aflcc); + aflcc->have_lsan = 1; } - if (getenv("AFL_NO_BUILTIN") || getenv("AFL_LLVM_LAF_TRANSFORM_COMPARES") || - getenv("LAF_TRANSFORM_COMPARES") || getenv("AFL_LLVM_LAF_ALL") || - lto_mode) { + if (getenv("AFL_USE_CFISAN") || aflcc->have_cfisan) { - cc_params[cc_par_cnt++] = "-fno-builtin-strcmp"; - cc_params[cc_par_cnt++] = "-fno-builtin-strncmp"; - cc_params[cc_par_cnt++] = "-fno-builtin-strcasecmp"; - cc_params[cc_par_cnt++] = "-fno-builtin-strncasecmp"; - cc_params[cc_par_cnt++] = "-fno-builtin-memcmp"; - cc_params[cc_par_cnt++] = "-fno-builtin-bcmp"; - cc_params[cc_par_cnt++] = "-fno-builtin-strstr"; - cc_params[cc_par_cnt++] = "-fno-builtin-strcasestr"; + if (aflcc->compiler_mode == GCC_PLUGIN || aflcc->compiler_mode == GCC) { - } + if (!aflcc->have_fcf) { insert_param(aflcc, "-fcf-protection=full"); } -#if defined(USEMMAP) && !defined(__HAIKU__) && !__APPLE__ - if (!have_c) cc_params[cc_par_cnt++] = "-lrt"; -#endif + } else { - cc_params[cc_par_cnt++] = "-D__AFL_COMPILER=1"; - cc_params[cc_par_cnt++] = "-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"; + if (!aflcc->lto_mode && !aflcc->have_flto) { - /* As documented in instrumentation/README.persistent_mode.md, deferred - forkserver initialization and persistent mode are not available in afl-gcc - and afl-clang. */ - if (compiler_mode != GCC && compiler_mode != CLANG) { + uint32_t i = 0, found = 0; + while (envp[i] != NULL && !found) { - cc_params[cc_par_cnt++] = "-D__AFL_HAVE_MANUAL_CONTROL=1"; + if (strncmp("-flto", envp[i++], 5) == 0) found = 1; - /* When the user tries to use persistent or deferred forkserver modes by - appending a single line to the program, we want to reliably inject a - signature into the binary (to be picked up by afl-fuzz) and we want - to call a function from the runtime .o file. This is unnecessarily - painful for three reasons: + } - 1) We need to convince the compiler not to optimize out the signature. - This is done with __attribute__((used)). + if (!found) { insert_param(aflcc, "-flto"); } + aflcc->have_flto = 1; - 2) We need to convince the linker, when called with -Wl,--gc-sections, - not to do the same. This is done by forcing an assignment to a - 'volatile' pointer. + } - 3) We need to declare __afl_persistent_loop() in the global namespace, - but doing this within a method in a class is hard - :: and extern "C" - are forbidden and __attribute__((alias(...))) doesn't work. Hence the - __asm__ aliasing trick. + if (!aflcc->have_cfisan) { insert_param(aflcc, "-fsanitize=cfi"); } - */ + if (!aflcc->have_hidden) { - cc_params[cc_par_cnt++] = - "-D__AFL_FUZZ_INIT()=" - "int __afl_sharedmem_fuzzing = 1;" - "extern unsigned int *__afl_fuzz_len;" - "extern unsigned char *__afl_fuzz_ptr;" - "unsigned char __afl_fuzz_alt[1048576];" - "unsigned char *__afl_fuzz_alt_ptr = __afl_fuzz_alt;"; + insert_param(aflcc, "-fvisibility=hidden"); + aflcc->have_hidden = 1; - } + } - if (plusplus_mode) { + aflcc->have_cfisan = 1; - cc_params[cc_par_cnt++] = - "-D__AFL_COVERAGE()=int __afl_selective_coverage = 1;" - "extern \"C\" void __afl_coverage_discard();" - "extern \"C\" void __afl_coverage_skip();" - "extern \"C\" void __afl_coverage_on();" - "extern \"C\" void __afl_coverage_off();"; + } - } else { + } - cc_params[cc_par_cnt++] = - "-D__AFL_COVERAGE()=int __afl_selective_coverage = 1;" - "void __afl_coverage_discard();" - "void __afl_coverage_skip();" - "void __afl_coverage_on();" - "void __afl_coverage_off();"; +} - } +/* Add params to enable LLVM SanCov, the native PCGUARD */ +void add_native_pcguard(aflcc_state_t *aflcc) { - cc_params[cc_par_cnt++] = - "-D__AFL_COVERAGE_START_OFF()=int __afl_selective_coverage_start_off = " - "1;"; - cc_params[cc_par_cnt++] = "-D__AFL_COVERAGE_ON()=__afl_coverage_on()"; - cc_params[cc_par_cnt++] = "-D__AFL_COVERAGE_OFF()=__afl_coverage_off()"; - cc_params[cc_par_cnt++] = - "-D__AFL_COVERAGE_DISCARD()=__afl_coverage_discard()"; - cc_params[cc_par_cnt++] = "-D__AFL_COVERAGE_SKIP()=__afl_coverage_skip()"; - cc_params[cc_par_cnt++] = - "-D__AFL_FUZZ_TESTCASE_BUF=(__afl_fuzz_ptr ? __afl_fuzz_ptr : " - "__afl_fuzz_alt_ptr)"; - cc_params[cc_par_cnt++] = - "-D__AFL_FUZZ_TESTCASE_LEN=(__afl_fuzz_ptr ? *__afl_fuzz_len : " - "(*__afl_fuzz_len = read(0, __afl_fuzz_alt_ptr, 1048576)) == 0xffffffff " - "? 0 : *__afl_fuzz_len)"; + /* If there is a rust ASan runtime on the command line, it is likely we're + * linking from rust and adding native flags requiring the sanitizer runtime + * will trigger native clang to add yet another runtime, causing linker + * errors. For now we shouldn't add instrumentation here, we're linking + * anyway. + */ + if (aflcc->have_rust_asanrt) { return; } - if (compiler_mode != GCC && compiler_mode != CLANG) { + /* If llvm-config doesn't figure out LLVM_MAJOR, just + go on anyway and let compiler complain if doesn't work. */ - cc_params[cc_par_cnt++] = - "-D__AFL_LOOP(_A)=" - "({ static volatile const char *_B __attribute__((used,unused)); " - " _B = (const char*)\"" PERSIST_SIG - "\"; " - "extern int __afl_connected;" -#ifdef __APPLE__ - "__attribute__((visibility(\"default\"))) " - "int _L(unsigned int) __asm__(\"___afl_persistent_loop\"); " -#else - "__attribute__((visibility(\"default\"))) " - "int _L(unsigned int) __asm__(\"__afl_persistent_loop\"); " -#endif /* ^__APPLE__ */ - // if afl is connected, we run _A times, else once. - "_L(__afl_connected ? _A : 1); })"; - - cc_params[cc_par_cnt++] = - "-D__AFL_INIT()=" - "do { static volatile const char *_A __attribute__((used,unused)); " - " _A = (const char*)\"" DEFER_SIG - "\"; " -#ifdef __APPLE__ - "__attribute__((visibility(\"default\"))) " - "void _I(void) __asm__(\"___afl_manual_init\"); " +#if LLVM_MAJOR > 0 && LLVM_MAJOR < 6 + FATAL("pcguard instrumentation with pc-table requires LLVM 6.0.1+"); #else - "__attribute__((visibility(\"default\"))) " - "void _I(void) __asm__(\"__afl_manual_init\"); " -#endif /* ^__APPLE__ */ - "_I(); } while (0)"; + #if LLVM_MAJOR == 0 + WARNF( + "pcguard instrumentation with pc-table requires LLVM 6.0.1+" + " otherwise the compiler will fail"); + #endif + if (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { - } + insert_param(aflcc, + "-fsanitize-coverage=trace-pc-guard,bb,no-prune,pc-table"); - if (x_set) { + } else { - cc_params[cc_par_cnt++] = "-x"; - cc_params[cc_par_cnt++] = "none"; + insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard,pc-table"); } - // prevent unnecessary build errors - if (compiler_mode != GCC_PLUGIN && compiler_mode != GCC) { +#endif - cc_params[cc_par_cnt++] = "-Wno-unused-command-line-argument"; +} - } +/* + Add params to launch our optimized PCGUARD on request. + It will fallback to use the native PCGUARD in some cases. If so, plz + bear in mind that instrument_mode will be set to INSTRUMENT_LLVMNATIVE. +*/ +void add_optimized_pcguard(aflcc_state_t *aflcc) { - if (preprocessor_only || have_c || !non_dash) { +#if LLVM_MAJOR >= 13 + #if defined __ANDROID__ || ANDROID - /* In the preprocessor_only case (-E), we are not actually compiling at - all but requesting the compiler to output preprocessed sources only. - We must not add the runtime in this case because the compiler will - simply output its binary content back on stdout, breaking any build - systems that rely on a separate source preprocessing step. */ - cc_params[cc_par_cnt] = NULL; - return; + insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard"); + aflcc->instrument_mode = INSTRUMENT_LLVMNATIVE; - } + #else -#ifndef __ANDROID__ + if (aflcc->have_instr_list) { - if (compiler_mode != GCC && compiler_mode != CLANG) { + if (!be_quiet) + SAYF( + "Using unoptimized trace-pc-guard, due usage of " + "-fsanitize-coverage-allow/denylist, you can use " + "AFL_LLVM_ALLOWLIST/AFL_LLVM_DENYLIST instead.\n"); - switch (bit_mode) { + insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard"); + aflcc->instrument_mode = INSTRUMENT_LLVMNATIVE; - case 0: - if (!shared_linking && !partial_linking) - cc_params[cc_par_cnt++] = - alloc_printf("%s/afl-compiler-rt.o", obj_path); - if (lto_mode) - cc_params[cc_par_cnt++] = - alloc_printf("%s/afl-llvm-rt-lto.o", obj_path); - break; + } else { - case 32: - if (!shared_linking && !partial_linking) { + /* Since LLVM_MAJOR >= 13 we use new pass manager */ + #if LLVM_MAJOR < 16 + insert_param(aflcc, "-fexperimental-new-pass-manager"); + #endif + insert_object(aflcc, "SanitizerCoveragePCGUARD.so", "-fpass-plugin=%s", 0); - cc_params[cc_par_cnt++] = - alloc_printf("%s/afl-compiler-rt-32.o", obj_path); - if (access(cc_params[cc_par_cnt - 1], R_OK)) - FATAL("-m32 is not supported by your compiler"); + } - } + #endif // defined __ANDROID__ || ANDROID +#else // LLVM_MAJOR < 13 + #if LLVM_MAJOR >= 4 + + if (!be_quiet) + SAYF( + "Using unoptimized trace-pc-guard, upgrade to LLVM 13+ for " + "enhanced version.\n"); + insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard"); + aflcc->instrument_mode = INSTRUMENT_LLVMNATIVE; - if (lto_mode) { + #else - cc_params[cc_par_cnt++] = - alloc_printf("%s/afl-llvm-rt-lto-32.o", obj_path); - if (access(cc_params[cc_par_cnt - 1], R_OK)) - FATAL("-m32 is not supported by your compiler"); + FATAL("pcguard instrumentation requires LLVM 4.0.1+"); - } + #endif +#endif - break; +} - case 64: - if (!shared_linking && !partial_linking) { +/** About -fsanitize -----END----- **/ - cc_params[cc_par_cnt++] = - alloc_printf("%s/afl-compiler-rt-64.o", obj_path); - if (access(cc_params[cc_par_cnt - 1], R_OK)) - FATAL("-m64 is not supported by your compiler"); +/** Linking behaviors -----BEGIN----- **/ - } +/* + Parse and process possible linking stage related args, + return PARAM_MISS if nothing matched. +*/ +param_st parse_linking_params(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan, + u8 *skip_next, char **argv) { - if (lto_mode) { + if (aflcc->lto_mode && !strncmp(cur_argv, "-flto=thin", 10)) { - cc_params[cc_par_cnt++] = - alloc_printf("%s/afl-llvm-rt-lto-64.o", obj_path); - if (access(cc_params[cc_par_cnt - 1], R_OK)) - FATAL("-m64 is not supported by your compiler"); + FATAL( + "afl-clang-lto cannot work with -flto=thin. Switch to -flto=full or " + "use afl-clang-fast!"); - } + } - break; + param_st final_ = PARAM_MISS; - } + if (!strcmp(cur_argv, "-shared") || !strcmp(cur_argv, "-dynamiclib")) { - #if !defined(__APPLE__) && !defined(__sun) - if (!shared_linking && !partial_linking) - cc_params[cc_par_cnt++] = - alloc_printf("-Wl,--dynamic-list=%s/dynamic_list.txt", obj_path); - #endif + if (scan) { - #if defined(__APPLE__) - if (shared_linking || partial_linking) { + aflcc->shared_linking = 1; + final_ = PARAM_SCAN; - cc_params[cc_par_cnt++] = "-Wl,-U"; - cc_params[cc_par_cnt++] = "-Wl,___afl_area_ptr"; - cc_params[cc_par_cnt++] = "-Wl,-U"; - cc_params[cc_par_cnt++] = "-Wl,___sanitizer_cov_trace_pc_guard_init"; + } else { + + final_ = PARAM_KEEP; } - #endif + } else if (!strcmp(cur_argv, "-Wl,-r") || !strcmp(cur_argv, "-Wl,-i") || - } + !strcmp(cur_argv, "-Wl,--relocatable") || + !strcmp(cur_argv, "-r") || !strcmp(cur_argv, "--relocatable")) { - #if defined(USEMMAP) && !defined(__HAIKU__) && !__APPLE__ - cc_params[cc_par_cnt++] = "-lrt"; - #endif + if (scan) { -#endif + aflcc->partial_linking = 1; + final_ = PARAM_SCAN; - cc_params[cc_par_cnt] = NULL; + } else { -} + final_ = PARAM_KEEP; -/* Main entry point */ + } -int main(int argc, char **argv, char **envp) { + } else if (!strncmp(cur_argv, "-fuse-ld=", 9) || - int i; - char *callname = argv[0], *ptr = NULL; + !strncmp(cur_argv, "--ld-path=", 10)) { - if (getenv("AFL_DEBUG")) { + if (scan) { - debug = 1; - if (strcmp(getenv("AFL_DEBUG"), "0") == 0) unsetenv("AFL_DEBUG"); + final_ = PARAM_SCAN; - } else if (getenv("AFL_QUIET")) + } else { - be_quiet = 1; + if (aflcc->lto_mode) + final_ = PARAM_DROP; + else + final_ = PARAM_KEEP; - if (getenv("AFL_LLVM_INSTRUMENT_FILE") || getenv("AFL_LLVM_WHITELIST") || - getenv("AFL_LLVM_ALLOWLIST") || getenv("AFL_LLVM_DENYLIST") || - getenv("AFL_LLVM_BLOCKLIST")) { + } - have_instr_env = 1; + } else if (!strcmp(cur_argv, "-Wl,-z,defs") || - } + !strcmp(cur_argv, "-Wl,--no-undefined") || + !strcmp(cur_argv, "-Wl,-no-undefined") || + !strcmp(cur_argv, "--no-undefined") || + strstr(cur_argv, "afl-compiler-rt") || + strstr(cur_argv, "afl-llvm-rt")) { - if (getenv("AFL_PASSTHROUGH") || getenv("AFL_NOOPT")) { + if (scan) { - passthrough = 1; - if (!debug) { be_quiet = 1; } + final_ = PARAM_SCAN; - } + } else { - if ((ptr = strrchr(callname, '/')) != NULL) callname = ptr + 1; - argvnull = (u8 *)argv[0]; - check_environment_vars(envp); + final_ = PARAM_DROP; - if ((ptr = find_object("as", argv[0])) != NULL) { + } - have_gcc = 1; - ck_free(ptr); + } else if (!strcmp(cur_argv, "-z") || !strcmp(cur_argv, "-Wl,-z")) { - } + u8 *param = *(argv + 1); + if (param && (!strcmp(param, "defs") || !strcmp(param, "-Wl,defs"))) { -#if (LLVM_MAJOR >= 3) + *skip_next = 1; - if ((ptr = find_object("SanitizerCoverageLTO.so", argv[0])) != NULL) { + if (scan) { - have_lto = 1; - ck_free(ptr); + final_ = PARAM_SCAN; - } + } else { - if ((ptr = find_object("cmplog-routines-pass.so", argv[0])) != NULL) { + final_ = PARAM_DROP; - have_llvm = 1; - ck_free(ptr); + } + + } } -#endif + // Try to warn user for some unsupported cases + if (scan && final_ == PARAM_MISS) { -#ifdef __ANDROID__ - have_llvm = 1; -#endif + u8 *ptr_ = NULL; - if ((ptr = find_object("afl-gcc-pass.so", argv[0])) != NULL) { + if (!strcmp(cur_argv, "-Xlinker") && (ptr_ = *(argv + 1))) { - have_gcc_plugin = 1; - ck_free(ptr); + if (!strcmp(ptr_, "defs")) { - } + WARNF("'-Xlinker' 'defs' detected. This may result in a bad link."); -#if (LLVM_MAJOR >= 3) + } else if (strstr(ptr_, "-no-undefined")) { - if (strncmp(callname, "afl-clang-fast", 14) == 0) { + WARNF( + "'-Xlinker' '%s' detected. The latter option may be dropped and " + "result in a bad link.", + ptr_); - compiler_mode = LLVM; + } - } else if (strncmp(callname, "afl-clang-lto", 13) == 0 || + } else if (!strncmp(cur_argv, "-Wl,", 4) && - strncmp(callname, "afl-lto", 7) == 0) { + (u8 *)strrchr(cur_argv, ',') != (cur_argv + 3)) { - compiler_mode = LTO; + ptr_ = cur_argv + 4; - } else + if (strstr(ptr_, "-shared") || strstr(ptr_, "-dynamiclib")) { -#endif - if (strncmp(callname, "afl-gcc-fast", 12) == 0 || + WARNF( + "'%s': multiple link options after '-Wl,' may break shared " + "linking.", + ptr_); - strncmp(callname, "afl-g++-fast", 12) == 0) { + } - compiler_mode = GCC_PLUGIN; + if (strstr(ptr_, "-r,") || strstr(ptr_, "-i,") || strstr(ptr_, ",-r") || + strstr(ptr_, ",-i") || strstr(ptr_, "--relocatable")) { - } else if (strncmp(callname, "afl-gcc", 7) == 0 || + WARNF( + "'%s': multiple link options after '-Wl,' may break partial " + "linking.", + ptr_); - strncmp(callname, "afl-g++", 7) == 0) { + } - compiler_mode = GCC; + if (strstr(ptr_, "defs") || strstr(ptr_, "no-undefined")) { - } else if (strcmp(callname, "afl-clang") == 0 || + WARNF( + "'%s': multiple link options after '-Wl,' may enable report " + "unresolved symbol references and result in a bad link.", + ptr_); - strcmp(callname, "afl-clang++") == 0) { + } - compiler_mode = CLANG; + } } - if ((ptr = getenv("AFL_CC_COMPILER"))) { + if (final_ == PARAM_KEEP) insert_param(aflcc, cur_argv); - if (compiler_mode) { + return final_; - if (!be_quiet) { +} - WARNF( - "\"AFL_CC_COMPILER\" is set but a specific compiler was already " - "selected by command line parameter or symlink, ignoring the " - "environment variable!"); +/* Add params to specify the linker used in LTO */ +void add_lto_linker(aflcc_state_t *aflcc) { - } + unsetenv("AFL_LD"); + unsetenv("AFL_LD_CALLER"); - } else { + u8 *ld_path = NULL; + if (getenv("AFL_REAL_LD")) { - if (strncasecmp(ptr, "LTO", 3) == 0) { + ld_path = strdup(getenv("AFL_REAL_LD")); - compiler_mode = LTO; + } else { - } else if (strncasecmp(ptr, "LLVM", 4) == 0) { + ld_path = strdup(AFL_REAL_LD); - compiler_mode = LLVM; + } - } else if (strncasecmp(ptr, "GCC_P", 5) == 0 || + if (!ld_path || !*ld_path) { - strncasecmp(ptr, "GCC-P", 5) == 0 || - strncasecmp(ptr, "GCCP", 4) == 0) { + if (ld_path) { - compiler_mode = GCC_PLUGIN; + // Freeing empty string + free(ld_path); - } else if (strcasecmp(ptr, "GCC") == 0) { + } - compiler_mode = GCC; + ld_path = strdup("ld.lld"); - } else + } - FATAL("Unknown AFL_CC_COMPILER mode: %s\n", ptr); + if (!ld_path) { PFATAL("Could not allocate mem for ld_path"); } +#if defined(AFL_CLANG_LDPATH) && LLVM_MAJOR >= 12 + insert_param(aflcc, alloc_printf("--ld-path=%s", ld_path)); +#else + insert_param(aflcc, alloc_printf("-fuse-ld=%s", ld_path)); +#endif + free(ld_path); - } +} - } +/* Add params to launch SanitizerCoverageLTO.so when linking */ +void add_lto_passes(aflcc_state_t *aflcc) { - if (strcmp(callname, "afl-clang") == 0 || - strcmp(callname, "afl-clang++") == 0) { +#if defined(AFL_CLANG_LDPATH) && LLVM_MAJOR >= 15 + // The NewPM implementation only works fully since LLVM 15. + insert_object(aflcc, "SanitizerCoverageLTO.so", "-Wl,--load-pass-plugin=%s", + 0); +#elif defined(AFL_CLANG_LDPATH) && LLVM_MAJOR >= 13 + insert_param(aflcc, "-Wl,--lto-legacy-pass-manager"); + insert_object(aflcc, "SanitizerCoverageLTO.so", "-Wl,-mllvm=-load=%s", 0); +#else + insert_param(aflcc, "-fno-experimental-new-pass-manager"); + insert_object(aflcc, "SanitizerCoverageLTO.so", "-Wl,-mllvm=-load=%s", 0); +#endif - clang_mode = 1; - compiler_mode = CLANG; + insert_param(aflcc, "-Wl,--allow-multiple-definition"); - if (strcmp(callname, "afl-clang++") == 0) { plusplus_mode = 1; } +} - } +/* Add params to link with libAFLDriver.a on request */ +static void add_aflpplib(aflcc_state_t *aflcc) { - for (i = 1; i < argc; i++) { + if (!aflcc->need_aflpplib) return; - if (strncmp(argv[i], "--afl", 5) == 0) { + u8 *afllib = find_object(aflcc, "libAFLDriver.a"); - if (!strcmp(argv[i], "--afl_noopt") || !strcmp(argv[i], "--afl-noopt")) { + if (!be_quiet) { - passthrough = 1; - argv[i] = "-g"; // we have to overwrite it, -g is always good - continue; + OKF("Found '-fsanitize=fuzzer', replacing with libAFLDriver.a"); - } + } - if (compiler_mode && !be_quiet) { + if (!afllib) { - WARNF( - "--afl-... compiler mode supersedes the AFL_CC_COMPILER and " - "symlink compiler selection!"); + if (!be_quiet) { - } + WARNF( + "Cannot find 'libAFLDriver.a' to replace '-fsanitize=fuzzer' in " + "the flags - this will fail!"); - ptr = argv[i]; - ptr += 5; - while (*ptr == '-') - ptr++; + } - if (strncasecmp(ptr, "LTO", 3) == 0) { + } else { - compiler_mode = LTO; + insert_param(aflcc, afllib); - } else if (strncasecmp(ptr, "LLVM", 4) == 0) { +#ifdef __APPLE__ + insert_param(aflcc, "-Wl,-undefined"); + insert_param(aflcc, "dynamic_lookup"); +#endif - compiler_mode = LLVM; + } - } else if (strncasecmp(ptr, "PCGUARD", 7) == 0 || +} - strncasecmp(ptr, "PC-GUARD", 8) == 0) { +/* Add params to link with runtimes depended by our instrumentation */ +void add_runtime(aflcc_state_t *aflcc) { - compiler_mode = LLVM; - instrument_mode = INSTRUMENT_PCGUARD; + if (aflcc->preprocessor_only || aflcc->have_c || !aflcc->non_dash) { - } else if (strcasecmp(ptr, "INSTRIM") == 0 || + /* In the preprocessor_only case (-E), we are not actually compiling at + all but requesting the compiler to output preprocessed sources only. + We must not add the runtime in this case because the compiler will + simply output its binary content back on stdout, breaking any build + systems that rely on a separate source preprocessing step. */ + return; - strcasecmp(ptr, "CFG") == 0) { + } - FATAL( - "InsTrim instrumentation was removed. Use a modern LLVM and " - "PCGUARD (default in afl-cc).\n"); + if (aflcc->compiler_mode != GCC_PLUGIN && aflcc->compiler_mode != GCC && + !getenv("AFL_LLVM_NO_RPATH")) { - } else if (strcasecmp(ptr, "AFL") == 0 || + // in case LLVM is installed not via a package manager or "make install" + // e.g. compiled download or compiled from github then its ./lib directory + // might not be in the search path. Add it if so. + const char *libdir = LLVM_LIBDIR; + if (aflcc->plusplus_mode && strlen(libdir) && strncmp(libdir, "/usr", 4) && + strncmp(libdir, "/lib", 4)) { - strcasecmp(ptr, "CLASSIC") == 0) { +#ifdef __APPLE__ + u8 *libdir_opt = strdup("-Wl,-rpath," LLVM_LIBDIR); +#else + u8 *libdir_opt = strdup("-Wl,-rpath=" LLVM_LIBDIR); +#endif + insert_param(aflcc, libdir_opt); - compiler_mode = LLVM; - instrument_mode = INSTRUMENT_CLASSIC; + } - } else if (strcasecmp(ptr, "LLVMNATIVE") == 0 || + } - strcasecmp(ptr, "NATIVE") == 0 || - strcasecmp(ptr, "LLVM-NATIVE") == 0) { +#ifndef __ANDROID__ - compiler_mode = LLVM; - instrument_mode = INSTRUMENT_LLVMNATIVE; + #define M32_ERR_MSG "-m32 is not supported by your compiler" + #define M64_ERR_MSG "-m64 is not supported by your compiler" - } else if (strncasecmp(ptr, "GCC_P", 5) == 0 || + if (aflcc->compiler_mode != GCC && aflcc->compiler_mode != CLANG) { - strncasecmp(ptr, "GCC-P", 5) == 0 || - strncasecmp(ptr, "GCCP", 4) == 0) { + switch (aflcc->bit_mode) { - compiler_mode = GCC_PLUGIN; + case 0: + if (!aflcc->shared_linking && !aflcc->partial_linking) + insert_object(aflcc, "afl-compiler-rt.o", 0, 0); + if (aflcc->lto_mode) insert_object(aflcc, "afl-llvm-rt-lto.o", 0, 0); + break; - } else if (strcasecmp(ptr, "GCC") == 0) { + case 32: + if (!aflcc->shared_linking && !aflcc->partial_linking) + insert_object(aflcc, "afl-compiler-rt-32.o", 0, M32_ERR_MSG); + if (aflcc->lto_mode) + insert_object(aflcc, "afl-llvm-rt-lto-32.o", 0, M32_ERR_MSG); + break; - compiler_mode = GCC; + case 64: + if (!aflcc->shared_linking && !aflcc->partial_linking) + insert_object(aflcc, "afl-compiler-rt-64.o", 0, M64_ERR_MSG); + if (aflcc->lto_mode) + insert_object(aflcc, "afl-llvm-rt-lto-64.o", 0, M64_ERR_MSG); + break; - } else if (strncasecmp(ptr, "CLANG", 5) == 0) { + } - compiler_mode = CLANG; + #if __AFL_CODE_COVERAGE + // Required for dladdr used in afl-compiler-rt.o + insert_param(aflcc, "-ldl"); + #endif - } else + #if !defined(__APPLE__) && !defined(__sun) + if (!aflcc->shared_linking && !aflcc->partial_linking) + insert_object(aflcc, "dynamic_list.txt", "-Wl,--dynamic-list=%s", 0); + #endif - FATAL("Unknown --afl-... compiler mode: %s\n", argv[i]); + #if defined(__APPLE__) + if (aflcc->shared_linking || aflcc->partial_linking) { + + insert_param(aflcc, "-Wl,-U"); + insert_param(aflcc, "-Wl,___afl_area_ptr"); + insert_param(aflcc, "-Wl,-U"); + insert_param(aflcc, "-Wl,___sanitizer_cov_trace_pc_guard_init"); } + #endif + } - if (strlen(callname) > 2 && - (strncmp(callname + strlen(callname) - 2, "++", 2) == 0 || - strstr(callname, "-g++") != NULL)) - plusplus_mode = 1; +#endif - if (getenv("USE_TRACE_PC") || getenv("AFL_USE_TRACE_PC") || - getenv("AFL_LLVM_USE_TRACE_PC") || getenv("AFL_TRACE_PC")) { + add_aflpplib(aflcc); - if (instrument_mode == 0) - instrument_mode = INSTRUMENT_PCGUARD; - else if (instrument_mode != INSTRUMENT_PCGUARD) - FATAL("you cannot set AFL_LLVM_INSTRUMENT and AFL_TRACE_PC together"); +#if defined(USEMMAP) && !defined(__HAIKU__) && !__APPLE__ + insert_param(aflcc, "-Wl,-lrt"); +#endif - } +} - if (have_instr_env && getenv("AFL_DONT_OPTIMIZE") && !be_quiet) { +/** Linking behaviors -----END----- **/ - WARNF( - "AFL_LLVM_ALLOWLIST/DENYLIST and AFL_DONT_OPTIMIZE cannot be combined " - "for file matching, only function matching!"); +/** Miscellaneous routines -----BEGIN----- **/ - } +/* + Add params to make compiler driver use our afl-as + as assembler, required by the vanilla instrumentation. +*/ +void add_assembler(aflcc_state_t *aflcc) { - if (getenv("AFL_LLVM_INSTRIM") || getenv("INSTRIM") || - getenv("INSTRIM_LIB")) { + u8 *afl_as = find_object(aflcc, "afl-as"); - FATAL( - "InsTrim instrumentation was removed. Use a modern LLVM and PCGUARD " - "(default in afl-cc).\n"); + if (!afl_as) FATAL("Cannot find 'afl-as'."); - } + u8 *slash = strrchr(afl_as, '/'); + if (slash) *slash = 0; - if (getenv("AFL_LLVM_CTX")) instrument_opt_mode |= INSTRUMENT_OPT_CTX; - if (getenv("AFL_LLVM_CALLER")) instrument_opt_mode |= INSTRUMENT_OPT_CALLER; + // Search for 'as' may be unreliable in some cases (see #2058) + // so use 'afl-as' instead, because 'as' is usually a symbolic link, + // or can be a renamed copy of 'afl-as' created in the same dir. + // Now we should verify if the compiler can find the 'as' we need. - if (getenv("AFL_LLVM_NGRAM_SIZE")) { +#define AFL_AS_ERR "(should be a symlink or copy of 'afl-as')" - instrument_opt_mode |= INSTRUMENT_OPT_NGRAM; - ngram_size = atoi(getenv("AFL_LLVM_NGRAM_SIZE")); - if (ngram_size < 2 || ngram_size > NGRAM_SIZE_MAX) - FATAL( - "NGRAM instrumentation mode must be between 2 and NGRAM_SIZE_MAX " - "(%u)", - NGRAM_SIZE_MAX); + u8 *afl_as_dup = alloc_printf("%s/as", afl_as); - } + int fd = open(afl_as_dup, O_RDONLY); + if (fd < 0) { PFATAL("Unable to open '%s' " AFL_AS_ERR, afl_as_dup); } - if (getenv("AFL_LLVM_CTX_K")) { + struct stat st; + if (fstat(fd, &st) < 0) { - ctx_k = atoi(getenv("AFL_LLVM_CTX_K")); - if (ctx_k < 1 || ctx_k > CTX_MAX_K) - FATAL("K-CTX instrumentation mode must be between 1 and CTX_MAX_K (%u)", - CTX_MAX_K); - if (ctx_k == 1) { + PFATAL("Unable to fstat '%s' " AFL_AS_ERR, afl_as_dup); - setenv("AFL_LLVM_CALLER", "1", 1); - unsetenv("AFL_LLVM_CTX_K"); - instrument_opt_mode |= INSTRUMENT_OPT_CALLER; + } - } else { + u32 f_len = st.st_size; - instrument_opt_mode |= INSTRUMENT_OPT_CTX_K; + u8 *f_data = mmap(0, f_len, PROT_READ, MAP_PRIVATE, fd, 0); + if (f_data == MAP_FAILED) { - } + PFATAL("Unable to mmap file '%s' " AFL_AS_ERR, afl_as_dup); } - if (getenv("AFL_LLVM_INSTRUMENT")) { + close(fd); - u8 *ptr2 = strtok(getenv("AFL_LLVM_INSTRUMENT"), ":,;"); + // "AFL_AS" is a const str passed to getenv in afl-as.c + if (!memmem(f_data, f_len, "AFL_AS", strlen("AFL_AS") + 1)) { - while (ptr2) { + FATAL( + "Looks like '%s' is not a valid symlink or copy of '%s/afl-as'. " + "It is a prerequisite to override system-wide 'as' for " + "instrumentation.", + afl_as_dup, afl_as); - if (strncasecmp(ptr2, "afl", strlen("afl")) == 0 || - strncasecmp(ptr2, "classic", strlen("classic")) == 0) { + } - if (instrument_mode == INSTRUMENT_LTO) { + if (munmap(f_data, f_len)) { PFATAL("unmap() failed"); } - instrument_mode = INSTRUMENT_CLASSIC; - lto_mode = 1; + ck_free(afl_as_dup); - } else if (!instrument_mode || instrument_mode == INSTRUMENT_AFL) { +#undef AFL_AS_ERR - instrument_mode = INSTRUMENT_AFL; + insert_param(aflcc, "-B"); + insert_param(aflcc, afl_as); - } else { + if (aflcc->compiler_mode == CLANG) insert_param(aflcc, "-no-integrated-as"); - FATAL("main instrumentation mode already set with %s", - instrument_mode_string[instrument_mode]); +} - } +/* Add params to launch the gcc plugins for instrumentation. */ +void add_gcc_plugin(aflcc_state_t *aflcc) { - } + if (aflcc->cmplog_mode) { - if (strncasecmp(ptr2, "pc-guard", strlen("pc-guard")) == 0 || - strncasecmp(ptr2, "pcguard", strlen("pcguard")) == 0) { + insert_object(aflcc, "afl-gcc-cmplog-pass.so", "-fplugin=%s", 0); + insert_object(aflcc, "afl-gcc-cmptrs-pass.so", "-fplugin=%s", 0); - if (!instrument_mode || instrument_mode == INSTRUMENT_PCGUARD) - instrument_mode = INSTRUMENT_PCGUARD; - else - FATAL("main instrumentation mode already set with %s", - instrument_mode_string[instrument_mode]); + } - } + insert_object(aflcc, "afl-gcc-pass.so", "-fplugin=%s", 0); - if (strncasecmp(ptr2, "llvmnative", strlen("llvmnative")) == 0 || - strncasecmp(ptr2, "llvm-native", strlen("llvm-native")) == 0) { + insert_param(aflcc, "-fno-if-conversion"); + insert_param(aflcc, "-fno-if-conversion2"); - if (!instrument_mode || instrument_mode == INSTRUMENT_LLVMNATIVE) - instrument_mode = INSTRUMENT_LLVMNATIVE; - else - FATAL("main instrumentation mode already set with %s", - instrument_mode_string[instrument_mode]); +} - } +/* Add some miscellaneous params required by our instrumentation. */ +void add_misc_params(aflcc_state_t *aflcc) { - if (strncasecmp(ptr2, "llvmcodecov", strlen("llvmcodecov")) == 0 || - strncasecmp(ptr2, "llvm-codecov", strlen("llvm-codecov")) == 0) { + if (getenv("AFL_NO_BUILTIN") || getenv("AFL_LLVM_LAF_TRANSFORM_COMPARES") || + getenv("AFL_LLVM_LAF_ALL") || getenv("AFL_LLVM_CMPLOG") || + aflcc->lto_mode) { + + insert_param(aflcc, "-fno-builtin-strcmp"); + insert_param(aflcc, "-fno-builtin-strncmp"); + insert_param(aflcc, "-fno-builtin-strcasecmp"); + insert_param(aflcc, "-fno-builtin-strncasecmp"); + insert_param(aflcc, "-fno-builtin-memcmp"); + insert_param(aflcc, "-fno-builtin-bcmp"); + insert_param(aflcc, "-fno-builtin-strstr"); + insert_param(aflcc, "-fno-builtin-strcasestr"); - if (!instrument_mode || instrument_mode == INSTRUMENT_LLVMNATIVE) { + } - instrument_mode = INSTRUMENT_LLVMNATIVE; - instrument_opt_mode |= INSTRUMENT_OPT_CODECOV; + if (!aflcc->have_pic) { insert_param(aflcc, "-fPIC"); } - } else { + if (getenv("AFL_HARDEN")) { - FATAL("main instrumentation mode already set with %s", - instrument_mode_string[instrument_mode]); + insert_param(aflcc, "-fstack-protector-all"); - } + if (!aflcc->fortify_set) add_defs_fortify(aflcc, 2); - } + } - if (strncasecmp(ptr2, "cfg", strlen("cfg")) == 0 || - strncasecmp(ptr2, "instrim", strlen("instrim")) == 0) { + if (!getenv("AFL_DONT_OPTIMIZE")) { - FATAL( - "InsTrim instrumentation was removed. Use a modern LLVM and " - "PCGUARD (default in afl-cc).\n"); + insert_param(aflcc, "-g"); + if (!aflcc->have_o) insert_param(aflcc, "-O3"); + if (!aflcc->have_unroll) insert_param(aflcc, "-funroll-loops"); + // if (strlen(aflcc->march_opt) > 1 && aflcc->march_opt[0] == '-') + // insert_param(aflcc, aflcc->march_opt); - } + } - if (strncasecmp(ptr2, "lto", strlen("lto")) == 0) { + if (aflcc->x_set) { - lto_mode = 1; - if (!instrument_mode || instrument_mode == INSTRUMENT_LTO) - instrument_mode = INSTRUMENT_LTO; - else - FATAL("main instrumentation mode already set with %s", - instrument_mode_string[instrument_mode]); + insert_param(aflcc, "-x"); + insert_param(aflcc, "none"); - } + } - if (strcasecmp(ptr2, "gcc") == 0) { +} - if (!instrument_mode || instrument_mode == INSTRUMENT_GCC) - instrument_mode = INSTRUMENT_GCC; - else if (instrument_mode != INSTRUMENT_GCC) - FATAL("main instrumentation mode already set with %s", - instrument_mode_string[instrument_mode]); - compiler_mode = GCC; +/* + Parse and process a variety of args under our matching rules, + return PARAM_MISS if nothing matched. +*/ +param_st parse_misc_params(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan) { - } + param_st final_ = PARAM_MISS; - if (strcasecmp(ptr2, "clang") == 0) { +// MACRO START +#define SCAN_KEEP(dst, src) \ + do { \ + \ + if (scan) { \ + \ + dst = src; \ + final_ = PARAM_SCAN; \ + \ + } else { \ + \ + final_ = PARAM_KEEP; \ + \ + } \ + \ + } while (0) - if (!instrument_mode || instrument_mode == INSTRUMENT_CLANG) - instrument_mode = INSTRUMENT_CLANG; - else if (instrument_mode != INSTRUMENT_CLANG) - FATAL("main instrumentation mode already set with %s", - instrument_mode_string[instrument_mode]); - compiler_mode = CLANG; + // MACRO END - } + if (!strncasecmp(cur_argv, "-fpic", 5)) { - if (strncasecmp(ptr2, "ctx-", strlen("ctx-")) == 0 || - strncasecmp(ptr2, "kctx-", strlen("c-ctx-")) == 0 || - strncasecmp(ptr2, "k-ctx-", strlen("k-ctx-")) == 0) { + SCAN_KEEP(aflcc->have_pic, 1); - u8 *ptr3 = ptr2; - while (*ptr3 && (*ptr3 < '0' || *ptr3 > '9')) - ptr3++; + } else if (!strcmp(cur_argv, "-m32") || - if (!*ptr3) { + !strcmp(cur_argv, "armv7a-linux-androideabi")) { - if ((ptr3 = getenv("AFL_LLVM_CTX_K")) == NULL) - FATAL( - "you must set the K-CTX K with (e.g. for value 2) " - "AFL_LLVM_INSTRUMENT=ctx-2"); + SCAN_KEEP(aflcc->bit_mode, 32); - } + } else if (!strcmp(cur_argv, "-m64")) { - ctx_k = atoi(ptr3); - if (ctx_k < 1 || ctx_k > CTX_MAX_K) - FATAL( - "K-CTX instrumentation option must be between 1 and CTX_MAX_K " - "(%u)", - CTX_MAX_K); + SCAN_KEEP(aflcc->bit_mode, 64); - if (ctx_k == 1) { + } else if (strstr(cur_argv, "FORTIFY_SOURCE")) { - instrument_opt_mode |= INSTRUMENT_OPT_CALLER; - setenv("AFL_LLVM_CALLER", "1", 1); - unsetenv("AFL_LLVM_CTX_K"); + SCAN_KEEP(aflcc->fortify_set, 1); - } else { + } else if (!strcmp(cur_argv, "-x")) { - instrument_opt_mode |= (INSTRUMENT_OPT_CTX_K); - u8 *ptr4 = alloc_printf("%u", ctx_k); - setenv("AFL_LLVM_CTX_K", ptr4, 1); + SCAN_KEEP(aflcc->x_set, 1); - } + } else if (!strcmp(cur_argv, "-E")) { - } + SCAN_KEEP(aflcc->preprocessor_only, 1); - if (strcasecmp(ptr2, "ctx") == 0) { + } else if (!strcmp(cur_argv, "--target=wasm32-wasi")) { - instrument_opt_mode |= INSTRUMENT_OPT_CTX; - setenv("AFL_LLVM_CTX", "1", 1); + SCAN_KEEP(aflcc->passthrough, 1); - } + } else if (!strcmp(cur_argv, "-c")) { - if (strncasecmp(ptr2, "caller", strlen("caller")) == 0) { + SCAN_KEEP(aflcc->have_c, 1); - instrument_opt_mode |= INSTRUMENT_OPT_CALLER; - setenv("AFL_LLVM_CALLER", "1", 1); + } else if (!strcmp(cur_argv, "-static-libasan")) { - } + SCAN_KEEP(aflcc->have_staticasan, 1); - if (strncasecmp(ptr2, "ngram", strlen("ngram")) == 0) { + } else if (strstr(cur_argv, "librustc") && strstr(cur_argv, "_rt.asan.a")) { - u8 *ptr3 = ptr2 + strlen("ngram"); - while (*ptr3 && (*ptr3 < '0' || *ptr3 > '9')) - ptr3++; + SCAN_KEEP(aflcc->have_rust_asanrt, 1); - if (!*ptr3) { + } else if (!strcmp(cur_argv, "-fno-omit-frame-pointer")) { - if ((ptr3 = getenv("AFL_LLVM_NGRAM_SIZE")) == NULL) - FATAL( - "you must set the NGRAM size with (e.g. for value 2) " - "AFL_LLVM_INSTRUMENT=ngram-2"); + SCAN_KEEP(aflcc->have_fp, 1); - } + } else if (!strcmp(cur_argv, "-fvisibility=hidden")) { - ngram_size = atoi(ptr3); - if (ngram_size < 2 || ngram_size > NGRAM_SIZE_MAX) - FATAL( - "NGRAM instrumentation option must be between 2 and " - "NGRAM_SIZE_MAX (%u)", - NGRAM_SIZE_MAX); - instrument_opt_mode |= (INSTRUMENT_OPT_NGRAM); - u8 *ptr4 = alloc_printf("%u", ngram_size); - setenv("AFL_LLVM_NGRAM_SIZE", ptr4, 1); + SCAN_KEEP(aflcc->have_hidden, 1); - } + } else if (!strcmp(cur_argv, "-flto") || !strcmp(cur_argv, "-flto=full")) { - ptr2 = strtok(NULL, ":,;"); + SCAN_KEEP(aflcc->have_flto, 1); - } + } else if (!strncmp(cur_argv, "-D_FORTIFY_SOURCE", - } + strlen("-D_FORTIFY_SOURCE"))) { - if ((instrument_opt_mode & INSTRUMENT_OPT_CTX) && - (instrument_opt_mode & INSTRUMENT_OPT_CALLER)) { + SCAN_KEEP(aflcc->have_fortify, 1); - FATAL("you cannot set CTX and CALLER together"); + } else if (!strncmp(cur_argv, "-fcf-protection", strlen("-fcf-protection"))) { - } + SCAN_KEEP(aflcc->have_cfisan, 1); - if ((instrument_opt_mode & INSTRUMENT_OPT_CTX) && - (instrument_opt_mode & INSTRUMENT_OPT_CTX_K)) { + } else if (!strncmp(cur_argv, "-O", 2)) { - FATAL("you cannot set CTX and K-CTX together"); + SCAN_KEEP(aflcc->have_o, 1); - } + } else if (!strncmp(cur_argv, "-funroll-loop", 13)) { - if ((instrument_opt_mode & INSTRUMENT_OPT_CALLER) && - (instrument_opt_mode & INSTRUMENT_OPT_CTX_K)) { + SCAN_KEEP(aflcc->have_unroll, 1); - FATAL("you cannot set CALLER and K-CTX together"); + } else if (!strncmp(cur_argv, "--afl", 5)) { - } - - if (instrument_opt_mode && instrument_mode == INSTRUMENT_DEFAULT && - (compiler_mode == LLVM || compiler_mode == UNSET)) { + if (scan) + final_ = PARAM_SCAN; + else + final_ = PARAM_DROP; - instrument_mode = INSTRUMENT_CLASSIC; - compiler_mode = LLVM; + } else if (!strncmp(cur_argv, "-fno-unroll", 11)) { - } + if (scan) + final_ = PARAM_SCAN; + else + final_ = PARAM_DROP; - if (!compiler_mode) { + } else if (!strcmp(cur_argv, "-pipe") && aflcc->compiler_mode == GCC_PLUGIN) { - // lto is not a default because outside of afl-cc RANLIB and AR have to - // be set to llvm versions so this would work - if (have_llvm) - compiler_mode = LLVM; - else if (have_gcc_plugin) - compiler_mode = GCC_PLUGIN; - else if (have_gcc) -#ifdef __APPLE__ - // on OSX clang masquerades as GCC - compiler_mode = CLANG; -#else - compiler_mode = GCC; -#endif - else if (have_lto) - compiler_mode = LTO; + if (scan) + final_ = PARAM_SCAN; else - FATAL("no compiler mode available"); + final_ = PARAM_DROP; - } + } else if (!strncmp(cur_argv, "-stdlib=", 8) && - if (compiler_mode == GCC) { + (aflcc->compiler_mode == GCC || + aflcc->compiler_mode == GCC_PLUGIN)) { - if (clang_mode) { + if (scan) { - instrument_mode = INSTRUMENT_CLANG; + final_ = PARAM_SCAN; } else { - instrument_mode = INSTRUMENT_GCC; + if (!be_quiet) WARNF("Found '%s' - stripping!", cur_argv); + final_ = PARAM_DROP; } - } + } else if (cur_argv[0] != '-') { - if (compiler_mode == CLANG) { + /* It's a weak, loose pattern, with very different purpose + than others. We handle it at last, cautiously and robustly. */ - instrument_mode = INSTRUMENT_CLANG; - setenv(CLANG_ENV_VAR, "1", 1); // used by afl-as + if (scan && cur_argv[0] != '@') // response file support + aflcc->non_dash = 1; } +#undef SCAN_KEEP + + if (final_ == PARAM_KEEP) insert_param(aflcc, cur_argv); + + return final_; + +} + +/** Miscellaneous routines -----END----- **/ + +/* Print help message on request */ +static void maybe_usage(aflcc_state_t *aflcc, int argc, char **argv) { + if (argc < 2 || strncmp(argv[1], "-h", 2) == 0) { printf("afl-cc" VERSION @@ -2091,36 +2793,44 @@ int main(int argc, char **argv, char **envp) { "-------------|\n" "MODES: NCC PERSIST DICT LAF " "CMPLOG SELECT\n" - " [LTO] llvm LTO: %s%s\n" - " PCGUARD DEFAULT yes yes yes yes yes " - " yes\n" - " CLASSIC yes yes yes yes yes " - " yes\n" - " [LLVM] llvm: %s%s\n" - " PCGUARD %s yes yes module yes yes " + " [LLVM] LLVM: %s%s\n" + " PCGUARD %s yes yes module yes yes " "yes\n" - " CLASSIC %s no yes module yes yes " + " NATIVE AVAILABLE no yes no no " + "part. yes\n" + " CLASSIC %s no yes module yes yes " "yes\n" " - NORMAL\n" " - CALLER\n" " - CTX\n" " - NGRAM-{2-16}\n" + " [LTO] LLVM LTO: %s%s\n" + " PCGUARD DEFAULT yes yes yes yes yes " + " yes\n" + " CLASSIC yes yes yes yes yes " + " yes\n" " [GCC_PLUGIN] gcc plugin: %s%s\n" " CLASSIC DEFAULT no yes no no no " "yes\n" " [GCC/CLANG] simple gcc/clang: %s%s\n" " CLASSIC DEFAULT no no no no no " "no\n\n", - have_lto ? "AVAILABLE" : "unavailable!", - compiler_mode == LTO ? " [SELECTED]" : "", - have_llvm ? "AVAILABLE" : "unavailable!", - compiler_mode == LLVM ? " [SELECTED]" : "", - LLVM_MAJOR >= 7 ? "DEFAULT" : " ", - LLVM_MAJOR >= 7 ? " " : "DEFAULT", - have_gcc_plugin ? "AVAILABLE" : "unavailable!", - compiler_mode == GCC_PLUGIN ? " [SELECTED]" : "", - have_gcc ? "AVAILABLE" : "unavailable!", - (compiler_mode == GCC || compiler_mode == CLANG) ? " [SELECTED]" : ""); + aflcc->have_llvm ? "AVAILABLE " : "unavailable!", + aflcc->compiler_mode == LLVM ? " [SELECTED]" : "", + aflcc->have_llvm ? "AVAILABLE " : "unavailable!", + aflcc->have_llvm ? "AVAILABLE " : "unavailable!", + aflcc->have_lto ? "AVAILABLE" : "unavailable!", + aflcc->compiler_mode == LTO ? " [SELECTED]" : "", + aflcc->have_gcc_plugin ? "AVAILABLE" : "unavailable!", + aflcc->compiler_mode == GCC_PLUGIN ? " [SELECTED]" : "", + aflcc->have_gcc && aflcc->have_clang + ? "AVAILABLE" + : (aflcc->have_gcc + ? "GCC ONLY " + : (aflcc->have_clang ? "CLANG ONLY" : "unavailable!")), + (aflcc->compiler_mode == GCC || aflcc->compiler_mode == CLANG) + ? " [SELECTED]" + : ""); SAYF( "Modes:\n" @@ -2166,7 +2876,7 @@ int main(int argc, char **argv, char **envp) { " (instrumentation/README.lto.md)\n" " PERSIST: persistent mode support [code] (huge speed increase!)\n" " (instrumentation/README.persistent_mode.md)\n" - " DICT: dictionary in the target [yes=automatic or llvm module " + " DICT: dictionary in the target [yes=automatic or LLVM module " "pass]\n" " (instrumentation/README.lto.md + " "instrumentation/README.llvm.md)\n" @@ -2194,7 +2904,7 @@ int main(int argc, char **argv, char **envp) { " AFL_DONT_OPTIMIZE: disable optimization instead of -O3\n" " AFL_NO_BUILTIN: no builtins for string compare functions (for " "libtokencap.so)\n" - " AFL_NOOP: behave like a normal compiler (to pass configure " + " AFL_NOOPT: behave like a normal compiler (to pass configure " "tests)\n" " AFL_PATH: path to instrumenting pass and runtime " "(afl-compiler-rt.*o)\n" @@ -2209,7 +2919,7 @@ int main(int argc, char **argv, char **envp) { " AFL_USE_TSAN: activate thread sanitizer\n" " AFL_USE_LSAN: activate leak-checker sanitizer\n"); - if (have_gcc_plugin) + if (aflcc->have_gcc_plugin) SAYF( "\nGCC Plugin-specific environment variables:\n" " AFL_GCC_CMPLOG: log operands of comparisons (RedQueen mutator)\n" @@ -2225,7 +2935,7 @@ int main(int argc, char **argv, char **envp) { #define COUNTER_BEHAVIOUR \ " AFL_LLVM_NOT_ZERO: use cycling trace counters that skip zero\n" #endif - if (have_llvm) + if (aflcc->have_llvm) SAYF( "\nLLVM/LTO/afl-clang-fast/afl-clang-lto specific environment " "variables:\n" @@ -2238,6 +2948,10 @@ int main(int argc, char **argv, char **envp) { "comparisons\n" " AFL_LLVM_DICT2FILE_NO_MAIN: skip parsing main() for the " "dictionary\n" + " AFL_LLVM_INJECTIONS_ALL: enables all injections hooking\n" + " AFL_LLVM_INJECTIONS_SQL: enables SQL injections hooking\n" + " AFL_LLVM_INJECTIONS_LDAP: enables LDAP injections hooking\n" + " AFL_LLVM_INJECTIONS_XSS: enables XSS injections hooking\n" " AFL_LLVM_LAF_ALL: enables all LAF splits/transforms\n" " AFL_LLVM_LAF_SPLIT_COMPARES: enable cascaded comparisons\n" " AFL_LLVM_LAF_SPLIT_COMPARES_BITW: size limit (default 8)\n" @@ -2249,7 +2963,7 @@ int main(int argc, char **argv, char **envp) { "instrument allow/\n" " deny listing (selective instrumentation)\n"); - if (have_llvm) + if (aflcc->have_llvm) SAYF( " AFL_LLVM_CMPLOG: log operands of comparisons (RedQueen " "mutator)\n" @@ -2263,10 +2977,12 @@ int main(int argc, char **argv, char **envp) { " AFL_LLVM_CTX: use full context sensitive coverage (for " "CLASSIC)\n" " AFL_LLVM_NGRAM_SIZE: use ngram prev_loc count coverage (for " - "CLASSIC)\n"); + "CLASSIC)\n" + " AFL_LLVM_NO_RPATH: disable rpath setting for custom LLVM " + "locations\n"); #ifdef AFL_CLANG_FLTO - if (have_lto) + if (aflcc->have_lto) SAYF( "\nLTO/afl-clang-lto specific environment variables:\n" " AFL_LLVM_MAP_ADDR: use a fixed coverage map address (speed), " @@ -2274,12 +2990,13 @@ int main(int argc, char **argv, char **envp) { "0x10000\n" " AFL_LLVM_DOCUMENT_IDS: write all edge IDs and the corresponding " "functions\n" - " into this file\n" + " into this file (LTO mode)\n" + " AFL_LLVM_LTO_CALLER: activate CALLER/CTX instrumentation\n" + " AFL_LLVM_LTO_CALLER_DEPTH: skip how many empty functions\n" " AFL_LLVM_LTO_DONTWRITEID: don't write the highest ID used to a " "global var\n" " AFL_LLVM_LTO_STARTID: from which ID to start counting from for " - "a " - "bb\n" + "a bb\n" " AFL_REAL_LD: use this lld linker instead of the compiled in " "path\n" " AFL_LLVM_LTO_SKIPINIT: don't inject initialization code " @@ -2302,9 +3019,9 @@ int main(int argc, char **argv, char **envp) { "targets.\n\n"); #if (LLVM_MAJOR >= 3) - if (have_lto) + if (aflcc->have_lto) SAYF("afl-cc LTO with ld=%s %s\n", AFL_REAL_LD, AFL_CLANG_FLTO); - if (have_llvm) + if (aflcc->have_llvm) SAYF("afl-cc LLVM version %d using the binary path \"%s\".\n", LLVM_MAJOR, LLVM_BINDIR); #endif @@ -2330,209 +3047,498 @@ int main(int argc, char **argv, char **envp) { "AFL_LLVM_CMPLOG and " "AFL_LLVM_DICT2FILE+AFL_LLVM_DICT2FILE_NO_MAIN.\n\n"); + if (LLVM_MAJOR < 13) { + + SAYF( + "Warning: It is highly recommended to use at least LLVM version 13 " + "(or better, higher) rather than %d!\n\n", + LLVM_MAJOR); + + } + exit(1); } - if (compiler_mode == LTO) { +} - if (instrument_mode == 0 || instrument_mode == INSTRUMENT_LTO || - instrument_mode == INSTRUMENT_CFG || - instrument_mode == INSTRUMENT_PCGUARD) { +/* + Process params passed to afl-cc. + + We have two working modes, *scan* and *non-scan*. In scan mode, + the main task is to set some variables in aflcc according to current argv[i], + while in non-scan mode, is to choose keep or drop current argv[i]. + + We have several matching routines being called sequentially in the while-loop, + and each of them try to parse and match current argv[i] according to their own + rules. If one miss match, the next will then take over. In non-scan mode, each + argv[i] mis-matched by all the routines will be kept. + + These routines are: + 1. parse_misc_params + 2. parse_fsanitize + 3. parse_linking_params + 4. `if (*cur == '@') {...}`, i.e., parse response files +*/ +static void process_params(aflcc_state_t *aflcc, u8 scan, u32 argc, + char **argv) { - lto_mode = 1; - // force CFG - // if (!instrument_mode) { + // for (u32 x = 0; x < argc; ++x) fprintf(stderr, "[%u] %s\n", x, argv[x]); - instrument_mode = INSTRUMENT_PCGUARD; - // ptr = instrument_mode_string[instrument_mode]; - // } + /* Process the argument list. */ - } else if (instrument_mode == INSTRUMENT_CLASSIC) { + u8 skip_next = 0; + while (--argc) { - lto_mode = 1; + u8 *cur = *(++argv); - } else { + if (skip_next > 0) { - if (!be_quiet) { + skip_next--; + continue; - WARNF("afl-clang-lto called with mode %s, using that mode instead", - instrument_mode_string[instrument_mode]); + } + + if (PARAM_MISS != parse_misc_params(aflcc, cur, scan)) continue; + + if (PARAM_MISS != parse_fsanitize(aflcc, cur, scan)) continue; + + if (PARAM_MISS != parse_linking_params(aflcc, cur, scan, &skip_next, argv)) + continue; + + /* Response file support -----BEGIN----- + We have two choices - move everything to the command line or + rewrite the response files to temporary files and delete them + afterwards. We choose the first for easiness. + For clang, llvm::cl::ExpandResponseFiles does this, however it + only has C++ interface. And for gcc there is expandargv in libiberty, + written in C, but we can't simply copy-paste since its LGPL licensed. + So here we use an equivalent FSM as alternative, and try to be compatible + with the two above. See: + - https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html + - driver::expand_at_files in gcc.git/gcc/gcc.c + - expandargv in gcc.git/libiberty/argv.c + - llvm-project.git/clang/tools/driver/driver.cpp + - ExpandResponseFiles in + llvm-project.git/llvm/lib/Support/CommandLine.cpp + */ + if (*cur == '@') { + + u8 *filename = cur + 1; + if (aflcc->debug) { DEBUGF("response file=%s\n", filename); } + + // Check not found or empty? let the compiler complain if so. + FILE *f = fopen(filename, "r"); + if (!f) { + + if (!scan) insert_param(aflcc, cur); + continue; } - } + struct stat st; + if (fstat(fileno(f), &st) || !S_ISREG(st.st_mode) || st.st_size < 1) { - } + fclose(f); + if (!scan) insert_param(aflcc, cur); + continue; - if (instrument_mode == 0 && compiler_mode < GCC_PLUGIN) { + } -#if LLVM_MAJOR >= 7 - #if LLVM_MAJOR < 11 && (LLVM_MAJOR < 10 || LLVM_MINOR < 1) - if (have_instr_env) { + // Limit the number of response files, the max value + // just keep consistent with expandargv. Only do this in + // scan mode, and not touch rsp_count anymore in the next. + static u32 rsp_count = 2000; + if (scan) { - instrument_mode = INSTRUMENT_AFL; - if (!be_quiet) { + if (rsp_count == 0) FATAL("Too many response files provided!"); - WARNF( - "Switching to classic instrumentation because " - "AFL_LLVM_ALLOWLIST/DENYLIST does not work with PCGUARD < 10.0.1."); + --rsp_count; } - } else + // argc, argv acquired from this rsp file. Note that + // process_params ignores argv[0], we need to put a const "" here. + u32 argc_read = 1; + char **argv_read = ck_alloc(sizeof(char *)); + argv_read[0] = ""; - #endif - instrument_mode = INSTRUMENT_PCGUARD; + char *arg_buf = NULL; + u64 arg_len = 0; -#else - instrument_mode = INSTRUMENT_AFL; -#endif + enum fsm_state { - } + fsm_whitespace, // whitespace seen so far + fsm_double_quote, // have unpaired double quote + fsm_single_quote, // have unpaired single quote + fsm_backslash, // a backslash is seen with no unpaired quote + fsm_normal // a normal char is seen - if (instrument_opt_mode && compiler_mode != LLVM) - FATAL("CTX, CALLER and NGRAM can only be used in LLVM mode"); + }; - if (!instrument_opt_mode) { + // Workaround to append c to arg buffer, and append the buffer to argv +#define ARG_ALLOC(c) \ + do { \ + \ + ++arg_len; \ + arg_buf = ck_realloc(arg_buf, (arg_len + 1) * sizeof(char)); \ + arg_buf[arg_len] = '\0'; \ + arg_buf[arg_len - 1] = (char)c; \ + \ + } while (0) - if (lto_mode && instrument_mode == INSTRUMENT_CFG) - instrument_mode = INSTRUMENT_PCGUARD; - ptr = instrument_mode_string[instrument_mode]; +#define ARG_STORE() \ + do { \ + \ + ++argc_read; \ + argv_read = ck_realloc(argv_read, argc_read * sizeof(char *)); \ + argv_read[argc_read - 1] = arg_buf; \ + arg_buf = NULL; \ + arg_len = 0; \ + \ + } while (0) - } else { + int cur_chr = (int)' '; // init as whitespace, as a good start :) + enum fsm_state state_ = fsm_whitespace; - char *ptr2 = alloc_printf(" + NGRAM-%u", ngram_size); - char *ptr3 = alloc_printf(" + K-CTX-%u", ctx_k); + while (cur_chr != EOF) { - ptr = alloc_printf( - "%s%s%s%s%s", instrument_mode_string[instrument_mode], - (instrument_opt_mode & INSTRUMENT_OPT_CTX) ? " + CTX" : "", - (instrument_opt_mode & INSTRUMENT_OPT_CALLER) ? " + CALLER" : "", - (instrument_opt_mode & INSTRUMENT_OPT_NGRAM) ? ptr2 : "", - (instrument_opt_mode & INSTRUMENT_OPT_CTX_K) ? ptr3 : ""); + switch (state_) { - ck_free(ptr2); - ck_free(ptr3); + case fsm_whitespace: - } + if (arg_buf) { -#ifndef AFL_CLANG_FLTO - if (lto_mode) - FATAL( - "instrumentation mode LTO specified but LLVM support not available " - "(requires LLVM 11 or higher)"); -#endif + ARG_STORE(); + break; - if (instrument_opt_mode && instrument_opt_mode != INSTRUMENT_OPT_CODECOV && - instrument_mode != INSTRUMENT_CLASSIC) - FATAL( - "CALLER, CTX and NGRAM instrumentation options can only be used with " - "the LLVM CLASSIC instrumentation mode."); + } - if (getenv("AFL_LLVM_SKIP_NEVERZERO") && getenv("AFL_LLVM_NOT_ZERO")) - FATAL( - "AFL_LLVM_NOT_ZERO and AFL_LLVM_SKIP_NEVERZERO can not be set " - "together"); + if (isspace(cur_chr)) { -#if LLVM_MAJOR < 11 && (LLVM_MAJOR < 10 || LLVM_MINOR < 1) - if (instrument_mode == INSTRUMENT_PCGUARD && have_instr_env) { + cur_chr = fgetc(f); - FATAL( - "Instrumentation type PCGUARD does not support " - "AFL_LLVM_ALLOWLIST/DENYLIST! Use LLVM 10.0.1+ instead."); + } else if (cur_chr == (int)'\'') { - } + state_ = fsm_single_quote; + cur_chr = fgetc(f); -#endif + } else if (cur_chr == (int)'"') { - u8 *ptr2; + state_ = fsm_double_quote; + cur_chr = fgetc(f); - if ((ptr2 = getenv("AFL_LLVM_DICT2FILE")) != NULL && *ptr2 != '/') - FATAL("AFL_LLVM_DICT2FILE must be set to an absolute file path"); + } else if (cur_chr == (int)'\\') { - if ((isatty(2) && !be_quiet) || debug) { + state_ = fsm_backslash; + cur_chr = fgetc(f); - SAYF(cCYA - "afl-cc" VERSION cRST - " by Michal Zalewski, Laszlo Szekeres, Marc Heuse - mode: %s-%s\n", - compiler_mode_string[compiler_mode], ptr); + } else { - } + state_ = fsm_normal; - if (!be_quiet && (compiler_mode == GCC || compiler_mode == CLANG)) { + } - WARNF( - "You are using outdated instrumentation, install LLVM and/or " - "gcc-plugin and use afl-clang-fast/afl-clang-lto/afl-gcc-fast " - "instead!"); + break; + + case fsm_normal: + + if (isspace(cur_chr)) { + + state_ = fsm_whitespace; + + } else if (cur_chr == (int)'\'') { + + state_ = fsm_single_quote; + cur_chr = fgetc(f); + + } else if (cur_chr == (int)'\"') { + + state_ = fsm_double_quote; + cur_chr = fgetc(f); + + } else if (cur_chr == (int)'\\') { + + state_ = fsm_backslash; + cur_chr = fgetc(f); + + } else { + + ARG_ALLOC(cur_chr); + cur_chr = fgetc(f); + + } + + break; + + case fsm_backslash: + + ARG_ALLOC(cur_chr); + cur_chr = fgetc(f); + state_ = fsm_normal; + + break; + + case fsm_single_quote: + + if (cur_chr == (int)'\\') { + + cur_chr = fgetc(f); + if (cur_chr == EOF) break; + ARG_ALLOC(cur_chr); + + } else if (cur_chr == (int)'\'') { + + state_ = fsm_normal; + + } else { + + ARG_ALLOC(cur_chr); + + } + + cur_chr = fgetc(f); + break; + + case fsm_double_quote: + + if (cur_chr == (int)'\\') { + + cur_chr = fgetc(f); + if (cur_chr == EOF) break; + ARG_ALLOC(cur_chr); + + } else if (cur_chr == (int)'"') { + + state_ = fsm_normal; + + } else { + + ARG_ALLOC(cur_chr); + + } + + cur_chr = fgetc(f); + break; + + default: + break; + + } + + } + + if (arg_buf) { ARG_STORE(); } // save the pending arg after EOF + +#undef ARG_ALLOC +#undef ARG_STORE + + if (argc_read > 1) { process_params(aflcc, scan, argc_read, argv_read); } + + // We cannot free argv_read[] unless we don't need to keep any + // reference in cc_params. Never free argv[0], the const "". + if (scan) { + + while (argc_read > 1) + ck_free(argv_read[--argc_read]); + + ck_free(argv_read); + + } + + continue; + + } /* Response file support -----END----- */ + + if (!scan) insert_param(aflcc, cur); } - if (debug) { +} - DEBUGF("cd '%s';", getthecwd()); - for (i = 0; i < argc; i++) - SAYF(" '%s'", argv[i]); - SAYF("\n"); - fflush(stdout); - fflush(stderr); +/* Process each of the existing argv, also add a few new args. */ +static void edit_params(aflcc_state_t *aflcc, u32 argc, char **argv, + char **envp) { + + add_real_argv0(aflcc); + + // prevent unnecessary build errors + if (aflcc->compiler_mode != GCC_PLUGIN && aflcc->compiler_mode != GCC) { + + insert_param(aflcc, "-Wno-unused-command-line-argument"); } - if (getenv("AFL_LLVM_LAF_ALL")) { + if (aflcc->compiler_mode == GCC || aflcc->compiler_mode == CLANG) { - setenv("AFL_LLVM_LAF_SPLIT_SWITCHES", "1", 1); - setenv("AFL_LLVM_LAF_SPLIT_COMPARES", "1", 1); - setenv("AFL_LLVM_LAF_SPLIT_FLOATS", "1", 1); - setenv("AFL_LLVM_LAF_TRANSFORM_COMPARES", "1", 1); + add_assembler(aflcc); } - cmplog_mode = getenv("AFL_CMPLOG") || getenv("AFL_LLVM_CMPLOG") || - getenv("AFL_GCC_CMPLOG"); + if (aflcc->compiler_mode == GCC_PLUGIN) { add_gcc_plugin(aflcc); } -#if !defined(__ANDROID__) && !defined(ANDROID) - ptr = find_object("afl-compiler-rt.o", argv[0]); + if (aflcc->compiler_mode == LLVM || aflcc->compiler_mode == LTO) { - if (!ptr) { + if (aflcc->lto_mode && aflcc->have_instr_env) { - FATAL( - "Unable to find 'afl-compiler-rt.o'. Please set the AFL_PATH " - "environment variable."); + load_llvm_pass(aflcc, "afl-llvm-lto-instrumentlist.so"); - } + } - if (debug) { DEBUGF("rt=%s obj_path=%s\n", ptr, obj_path); } + if (getenv("AFL_LLVM_DICT2FILE")) { - ck_free(ptr); -#endif + load_llvm_pass(aflcc, "afl-llvm-dict2file.so"); - edit_params(argc, argv, envp); + } - if (debug) { + // laf + if (getenv("LAF_SPLIT_SWITCHES") || getenv("AFL_LLVM_LAF_SPLIT_SWITCHES")) { - DEBUGF("cd '%s';", getthecwd()); - for (i = 0; i < (s32)cc_par_cnt; i++) - SAYF(" '%s'", cc_params[i]); - SAYF("\n"); - fflush(stdout); - fflush(stderr); + load_llvm_pass(aflcc, "split-switches-pass.so"); + + } + + if (getenv("LAF_TRANSFORM_COMPARES") || + getenv("AFL_LLVM_LAF_TRANSFORM_COMPARES")) { + + load_llvm_pass(aflcc, "compare-transform-pass.so"); + + } + + if (getenv("LAF_SPLIT_COMPARES") || getenv("AFL_LLVM_LAF_SPLIT_COMPARES") || + getenv("AFL_LLVM_LAF_SPLIT_FLOATS")) { + + load_llvm_pass(aflcc, "split-compares-pass.so"); + + } + + // /laf + + if (aflcc->cmplog_mode) { + + insert_param(aflcc, "-fno-inline"); + + load_llvm_pass(aflcc, "cmplog-switches-pass.so"); + // reuse split switches from laf + load_llvm_pass(aflcc, "split-switches-pass.so"); + + } + + // #if LLVM_MAJOR >= 13 + // // Use the old pass manager in LLVM 14 which the AFL++ passes still + // use. insert_param(aflcc, "-flegacy-pass-manager"); + // #endif + + if (aflcc->lto_mode) { + + insert_param(aflcc, aflcc->lto_flag); + + if (!aflcc->have_c) { + + add_lto_linker(aflcc); + add_lto_passes(aflcc); + + } + + } else { + + if (aflcc->instrument_mode == INSTRUMENT_PCGUARD) { + + add_optimized_pcguard(aflcc); + + } else if (aflcc->instrument_mode == INSTRUMENT_LLVMNATIVE) { + + add_native_pcguard(aflcc); + + } else { + + load_llvm_pass(aflcc, "afl-llvm-pass.so"); + + } + + } + + if (aflcc->cmplog_mode) { + + load_llvm_pass(aflcc, "cmplog-instructions-pass.so"); + load_llvm_pass(aflcc, "cmplog-routines-pass.so"); + + } + + if (getenv("AFL_LLVM_INJECTIONS_ALL") || + getenv("AFL_LLVM_INJECTIONS_SQL") || + getenv("AFL_LLVM_INJECTIONS_LDAP") || + getenv("AFL_LLVM_INJECTIONS_XSS")) { + + load_llvm_pass(aflcc, "injection-pass.so"); + + } + + // insert_param(aflcc, "-Qunused-arguments"); } - if (passthrough) { + /* Inspect the command line parameters. */ + + process_params(aflcc, 0, argc, argv); + + add_sanitizers(aflcc, envp); + + add_misc_params(aflcc); + + add_defs_common(aflcc); + add_defs_selective_instr(aflcc); + add_defs_persistent_mode(aflcc); + + add_runtime(aflcc); + + insert_param(aflcc, NULL); + +} + +/* Main entry point */ +int main(int argc, char **argv, char **envp) { + + aflcc_state_t *aflcc = malloc(sizeof(aflcc_state_t)); + aflcc_state_init(aflcc, (u8 *)argv[0]); + + check_environment_vars(envp); + + find_built_deps(aflcc); + + compiler_mode_by_callname(aflcc); + compiler_mode_by_environ(aflcc); + compiler_mode_by_cmdline(aflcc, argc, argv); + + instrument_mode_by_environ(aflcc); + + mode_final_checkout(aflcc, argc, argv); + + process_params(aflcc, 1, argc, argv); + + maybe_usage(aflcc, argc, argv); + + mode_notification(aflcc); + + if (aflcc->debug) debugf_args(argc, argv); + + edit_params(aflcc, argc, argv, envp); + + if (aflcc->debug) + debugf_args((s32)aflcc->cc_par_cnt, (char **)aflcc->cc_params); + + if (aflcc->passthrough) { - argv[0] = cc_params[0]; - execvp(cc_params[0], (char **)argv); + argv[0] = aflcc->cc_params[0]; + execvp(aflcc->cc_params[0], (char **)argv); } else { - execvp(cc_params[0], (char **)cc_params); + execvp(aflcc->cc_params[0], (char **)aflcc->cc_params); } - FATAL("Oops, failed to execute '%s' - check your PATH", cc_params[0]); + FATAL("Oops, failed to execute '%s' - check your PATH", aflcc->cc_params[0]); return 0; diff --git a/src/afl-common.c b/src/afl-common.c index a5c48e80..9a27824d 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -34,6 +34,7 @@ #endif #include <string.h> #include <strings.h> +#include <time.h> #include <math.h> #include <sys/mman.h> @@ -58,6 +59,27 @@ u8 last_intr = 0; #define AFL_PATH "/usr/local/lib/afl/" #endif +/* - Some BSD (i.e.: FreeBSD) offer the FAST clock source as + * equivalent to Linux COARSE clock source. Aliasing COARSE to + * FAST on such systems when COARSE is not already defined. + * - macOS has no support of CLOCK_MONOTONIC_COARSE clock type. + */ +#if defined(OS_DARWIN) || defined(OS_SUNOS) || defined(__APPLE__) || \ + defined(__sun) || defined(__NetBSD__) + #define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC +#elif defined(OS_FREEBSD) + #define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC_FAST +#endif + +/* Convert seconds to milliseconds. */ +#define SEC_TO_MS(sec) ((sec) * 1000) +/* Convert seconds to microseconds. */ +#define SEC_TO_US(sec) ((sec) * 1000000) +/* Convert nanoseconds to milliseconds. */ +#define NS_TO_MS(ns) ((ns) / 1000000) +/* Convert nanoseconds to microseconds. */ +#define NS_TO_US(ns) ((ns) / 1000) + void *afl_memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen) { @@ -86,9 +108,10 @@ void set_sanitizer_defaults() { u8 *have_lsan_options = getenv("LSAN_OPTIONS"); u8 have_san_options = 0; u8 default_options[1024] = - "detect_odr_violation=0:abort_on_error=1:symbolize=0:allocator_may_" - "return_null=1:handle_segv=0:handle_sigbus=0:handle_abort=0:handle_" - "sigfpe=0:handle_sigill=0:"; + "detect_odr_violation=0:abort_on_error=1:symbolize=0:" + "allocator_may_return_null=1:handle_segv=0:handle_sigbus=0:" + "handle_abort=0:handle_sigfpe=0:handle_sigill=0:" + "detect_stack_use_after_return=0:check_initialization_order=0:"; if (have_asan_options || have_ubsan_options || have_msan_options || have_lsan_options) { @@ -98,12 +121,27 @@ void set_sanitizer_defaults() { } /* LSAN does not support abort_on_error=1. (is this still true??) */ + u8 should_detect_leaks = 0; if (!have_lsan_options) { u8 buf[2048] = ""; if (!have_san_options) { strcpy(buf, default_options); } - strcat(buf, "exitcode=" STRINGIFY(LSAN_ERROR) ":fast_unwind_on_malloc=0:print_suppressions=0:detect_leaks=1:malloc_context_size=30:"); + if (have_asan_options) { + + if (NULL != strstr(have_asan_options, "detect_leaks=0")) { + + strcat(buf, "exitcode=" STRINGIFY(LSAN_ERROR) ":fast_unwind_on_malloc=0:print_suppressions=0:detect_leaks=0:malloc_context_size=0:"); + + } else { + + should_detect_leaks = 1; + strcat(buf, "exitcode=" STRINGIFY(LSAN_ERROR) ":fast_unwind_on_malloc=0:print_suppressions=0:detect_leaks=1:malloc_context_size=30:"); + + } + + } + setenv("LSAN_OPTIONS", buf, 1); } @@ -112,7 +150,15 @@ void set_sanitizer_defaults() { if (!have_lsan_options) { - strcat(default_options, "detect_leaks=0:malloc_context_size=0:"); + if (should_detect_leaks) { + + strcat(default_options, "detect_leaks=1:malloc_context_size=30:"); + + } else { + + strcat(default_options, "detect_leaks=0:malloc_context_size=0:"); + + } } @@ -403,7 +449,7 @@ u8 *find_binary(u8 *fname) { FATAL( "Unexpected overflow when processing ENV. This should never " - "happend."); + "had happened."); } @@ -949,14 +995,18 @@ void read_bitmap(u8 *fname, u8 *map, size_t len) { /* Get unix time in milliseconds */ -u64 get_cur_time(void) { +inline u64 get_cur_time(void) { + + struct timespec ts; + int rc = clock_gettime(CLOCK_MONOTONIC_COARSE, &ts); + if (rc == -1) { - struct timeval tv; - struct timezone tz; + PFATAL("Failed to obtain timestamp (errno = %i: %s)\n", errno, + strerror(errno)); - gettimeofday(&tv, &tz); + } - return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000); + return SEC_TO_MS((uint64_t)ts.tv_sec) + NS_TO_MS((uint64_t)ts.tv_nsec); } @@ -964,12 +1014,16 @@ u64 get_cur_time(void) { u64 get_cur_time_us(void) { - struct timeval tv; - struct timezone tz; + struct timespec ts; + int rc = clock_gettime(CLOCK_MONOTONIC_COARSE, &ts); + if (rc == -1) { - gettimeofday(&tv, &tz); + PFATAL("Failed to obtain timestamp (errno = %i: %s)\n", errno, + strerror(errno)); - return (tv.tv_sec * 1000000ULL) + tv.tv_usec; + } + + return SEC_TO_US((uint64_t)ts.tv_sec) + NS_TO_US((uint64_t)ts.tv_nsec); } @@ -1298,6 +1352,35 @@ u8 *u_stringify_time_diff(u8 *buf, u64 cur_ms, u64 event_ms) { } +/* Unsafe describe time delta as simple string. + Returns a pointer to buf for convenience. */ + +u8 *u_simplestring_time_diff(u8 *buf, u64 cur_ms, u64 event_ms) { + + if (!event_ms) { + + sprintf(buf, "00:00:00"); + + } else { + + u64 delta; + s32 t_d, t_h, t_m, t_s; + + delta = cur_ms - event_ms; + + t_d = delta / 1000 / 60 / 60 / 24; + t_h = (delta / 1000 / 60 / 60) % 24; + t_m = (delta / 1000 / 60) % 60; + t_s = (delta / 1000) % 60; + + sprintf(buf, "%d:%02d:%02d:%02d", t_d, t_h, t_m, t_s); + + } + + return buf; + +} + /* Reads the map size from ENV */ u32 get_map_size(void) { diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 30c8901c..beb6bdeb 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -7,13 +7,13 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> and Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,6 +27,9 @@ */ #include "config.h" +#ifdef AFL_PERSISTENT_RECORD + #include "afl-fuzz.h" +#endif #include "types.h" #include "debug.h" #include "common.h" @@ -129,6 +132,10 @@ nyx_plugin_handler_t *afl_load_libnyx_plugin(u8 *libnyx_binary) { plugin->nyx_remove_work_dir = dlsym(handle, "nyx_remove_work_dir"); if (plugin->nyx_remove_work_dir == NULL) { goto fail; } + plugin->nyx_config_set_aux_buffer_size = + dlsym(handle, "nyx_config_set_aux_buffer_size"); + if (plugin->nyx_config_set_aux_buffer_size == NULL) { goto fail; } + OKF("libnyx plugin is ready!"); return plugin; @@ -160,6 +167,8 @@ void afl_nyx_runner_kill(afl_forkserver_t *fsrv) { } + if (fsrv->nyx_log_fd >= 0) { close(fsrv->nyx_log_fd); } + } } @@ -214,6 +223,7 @@ void afl_fsrv_init(afl_forkserver_t *fsrv) { fsrv->nyx_bind_cpu_id = 0xFFFFFFFF; fsrv->nyx_use_tmp_workdir = false; fsrv->nyx_tmp_workdir_path = NULL; + fsrv->nyx_log_fd = -1; #endif // this structure needs default so we initialize it if this was not done @@ -265,6 +275,7 @@ void afl_fsrv_init_dup(afl_forkserver_t *fsrv_to, afl_forkserver_t *from) { fsrv_to->uses_crash_exitcode = from->uses_crash_exitcode; fsrv_to->crash_exitcode = from->crash_exitcode; fsrv_to->child_kill_signal = from->child_kill_signal; + fsrv_to->fsrv_kill_signal = from->fsrv_kill_signal; fsrv_to->debug = from->debug; // These are forkserver specific. @@ -381,7 +392,7 @@ static void afl_fauxsrv_execv(afl_forkserver_t *fsrv, char **argv) { while (1) { uint32_t was_killed; - int status; + u32 status; /* Wait for parent by reading from the pipe. Exit if read fails. */ @@ -516,7 +527,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, volatile u8 *stop_soon_p, u8 debug_child_output) { int st_pipe[2], ctl_pipe[2]; - s32 status; + u32 status; s32 rlen; char *ignore_autodict = getenv("AFL_NO_AUTODICT"); @@ -567,10 +578,26 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, void *nyx_config = fsrv->nyx_handlers->nyx_config_load(fsrv->target_path); fsrv->nyx_handlers->nyx_config_set_workdir_path(nyx_config, workdir_path); - fsrv->nyx_handlers->nyx_config_set_input_buffer_size(nyx_config, MAX_FILE); + fsrv->nyx_handlers->nyx_config_set_input_buffer_size(nyx_config, fsrv->max_length); fsrv->nyx_handlers->nyx_config_set_input_buffer_write_protection(nyx_config, true); + char *nyx_log_path = getenv("AFL_NYX_LOG"); + if (nyx_log_path) { + + fsrv->nyx_log_fd = + open(nyx_log_path, O_CREAT | O_TRUNC | O_WRONLY, DEFAULT_PERMISSION); + if (fsrv->nyx_log_fd < 0) { + + NYX_PRE_FATAL(fsrv, "AFL_NYX_LOG path could not be written"); + + } + + fsrv->nyx_handlers->nyx_config_set_hprintf_fd(nyx_config, + fsrv->nyx_log_fd); + + } + if (fsrv->nyx_standalone) { fsrv->nyx_handlers->nyx_config_set_process_role(nyx_config, StandAlone); @@ -589,23 +616,42 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, } - if (getenv("NYX_REUSE_SNAPSHOT") != NULL) { + if (getenv("AFL_NYX_AUX_SIZE") != NULL) { + + fsrv->nyx_aux_string_len = atoi(getenv("AFL_NYX_AUX_SIZE")); + + if (fsrv->nyx_handlers->nyx_config_set_aux_buffer_size( + nyx_config, fsrv->nyx_aux_string_len) != 1) { + + NYX_PRE_FATAL(fsrv, + "Invalid AFL_NYX_AUX_SIZE value set (must be a multiple " + "of 4096) ..."); + + } + + } else { + + fsrv->nyx_aux_string_len = 0x1000; + + } + + if (getenv("AFL_NYX_REUSE_SNAPSHOT") != NULL) { - if (access(getenv("NYX_REUSE_SNAPSHOT"), F_OK) == -1) { + if (access(getenv("AFL_NYX_REUSE_SNAPSHOT"), F_OK) == -1) { - NYX_PRE_FATAL(fsrv, "NYX_REUSE_SNAPSHOT path does not exist"); + NYX_PRE_FATAL(fsrv, "AFL_NYX_REUSE_SNAPSHOT path does not exist"); } /* stupid sanity check to avoid passing an empty or invalid snapshot * directory */ char *snapshot_file_path = - alloc_printf("%s/global.state", getenv("NYX_REUSE_SNAPSHOT")); + alloc_printf("%s/global.state", getenv("AFL_NYX_REUSE_SNAPSHOT")); if (access(snapshot_file_path, R_OK) == -1) { - NYX_PRE_FATAL( - fsrv, - "NYX_REUSE_SNAPSHOT path does not contain a valid Nyx snapshot"); + NYX_PRE_FATAL(fsrv, + "AFL_NYX_REUSE_SNAPSHOT path does not contain a valid " + "Nyx snapshot"); } @@ -617,13 +663,14 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, char *workdir_snapshot_path = alloc_printf("%s/workdir/snapshot", outdir_path_absolute); char *reuse_snapshot_path_real = - realpath(getenv("NYX_REUSE_SNAPSHOT"), NULL); + realpath(getenv("AFL_NYX_REUSE_SNAPSHOT"), NULL); if (strcmp(workdir_snapshot_path, reuse_snapshot_path_real) == 0) { - NYX_PRE_FATAL(fsrv, - "NYX_REUSE_SNAPSHOT path is located in current workdir " - "(use another output directory)"); + NYX_PRE_FATAL( + fsrv, + "AFL_NYX_REUSE_SNAPSHOT path is located in current workdir " + "(use another output directory)"); } @@ -631,12 +678,11 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, ck_free(workdir_snapshot_path); fsrv->nyx_handlers->nyx_config_set_reuse_snapshot_path( - nyx_config, getenv("NYX_REUSE_SNAPSHOT")); + nyx_config, getenv("AFL_NYX_REUSE_SNAPSHOT")); } - fsrv->nyx_runner = - fsrv->nyx_handlers->nyx_new(nyx_config, fsrv->nyx_bind_cpu_id); + fsrv->nyx_runner = fsrv->nyx_handlers->nyx_new(nyx_config, fsrv->nyx_id); ck_free(workdir_path); ck_free(outdir_path_absolute); @@ -653,27 +699,27 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, fsrv->nyx_handlers->nyx_get_bitmap_buffer(fsrv->nyx_runner); fsrv->nyx_handlers->nyx_option_set_reload_mode( - fsrv->nyx_runner, getenv("NYX_DISABLE_SNAPSHOT_MODE") == NULL); + fsrv->nyx_runner, getenv("AFL_NYX_DISABLE_SNAPSHOT_MODE") == NULL); fsrv->nyx_handlers->nyx_option_apply(fsrv->nyx_runner); fsrv->nyx_handlers->nyx_option_set_timeout(fsrv->nyx_runner, 2, 0); fsrv->nyx_handlers->nyx_option_apply(fsrv->nyx_runner); - fsrv->nyx_aux_string = malloc(0x1000); - memset(fsrv->nyx_aux_string, 0, 0x1000); + fsrv->nyx_aux_string = malloc(fsrv->nyx_aux_string_len); + memset(fsrv->nyx_aux_string, 0, fsrv->nyx_aux_string_len); /* dry run */ fsrv->nyx_handlers->nyx_set_afl_input(fsrv->nyx_runner, "INIT", 4); switch (fsrv->nyx_handlers->nyx_exec(fsrv->nyx_runner)) { case Abort: - NYX_PRE_FATAL(fsrv, "Error: Nyx abort occured..."); + NYX_PRE_FATAL(fsrv, "Error: Nyx abort occurred..."); break; case IoError: NYX_PRE_FATAL(fsrv, "Error: QEMU-Nyx has died..."); break; case Error: - NYX_PRE_FATAL(fsrv, "Error: Nyx runtime error has occured..."); + NYX_PRE_FATAL(fsrv, "Error: Nyx runtime error has occurred..."); break; default: break; @@ -681,7 +727,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, } /* autodict in Nyx mode */ - if (!ignore_autodict) { + if (!ignore_autodict && fsrv->add_extra_func) { char *x = alloc_printf("%s/workdir/dump/afl_autodict.txt", fsrv->out_dir_path); @@ -974,75 +1020,68 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (rlen == 4) { - if (!be_quiet) { OKF("All right - fork server is up."); } + /* + * The new fork server model works like this: + * Client: sends "AFLx" in little endian, with x being the forkserver + * protocol version. + * Server: replies with XOR of the message or exits with an error if it + * is not a supported version. + * Client: sends 32 bit of options and then sends all parameters of + * the options, one after another, increasing by option number. + * Ends with "AFLx". + * After the initial protocol version confirmation the server does not + * send any data anymore - except a future option requires this. + */ - if (getenv("AFL_DEBUG")) { + if ((status & FS_NEW_ERROR) == FS_NEW_ERROR) { - ACTF("Extended forkserver functions received (%08x).", status); + report_error_and_exit(status & 0x0000ffff); } - if ((status & FS_OPT_ERROR) == FS_OPT_ERROR) - report_error_and_exit(FS_OPT_GET_ERROR(status)); + if (status >= 0x41464c00 && status <= 0x41464cff) { - if ((status & FS_OPT_ENABLED) == FS_OPT_ENABLED) { + u32 version = status - 0x41464c00; - // workaround for recent AFL++ versions - if ((status & FS_OPT_OLD_AFLPP_WORKAROUND) == FS_OPT_OLD_AFLPP_WORKAROUND) - status = (status & 0xf0ffffff); + if (!version) { - if ((status & FS_OPT_NEWCMPLOG) == 0 && fsrv->cmplog_binary) { + FATAL( + "Fork server version is not assigned, this should not happen. " + "Recompile target."); - if (fsrv->qemu_mode || fsrv->frida_mode) { + } else if (version < FS_NEW_VERSION_MIN || version > FS_NEW_VERSION_MAX) { - report_error_and_exit(FS_ERROR_OLD_CMPLOG_QEMU); - - } else { - - report_error_and_exit(FS_ERROR_OLD_CMPLOG); - - } + FATAL( + "Fork server version is not not supported. Recompile the target."); } - if ((status & FS_OPT_SNAPSHOT) == FS_OPT_SNAPSHOT) { + u32 keep = status; + status ^= 0xffffffff; + if (write(fsrv->fsrv_ctl_fd, &status, 4) != 4) { - fsrv->snapshot = 1; - if (!be_quiet) { ACTF("Using SNAPSHOT feature."); } + FATAL("Writing to forkserver failed."); } - if ((status & FS_OPT_SHDMEM_FUZZ) == FS_OPT_SHDMEM_FUZZ) { - - if (fsrv->support_shmem_fuzz) { - - fsrv->use_shmem_fuzz = 1; - if (!be_quiet) { ACTF("Using SHARED MEMORY FUZZING feature."); } - - if ((status & FS_OPT_AUTODICT) == 0 || ignore_autodict) { - - u32 send_status = (FS_OPT_ENABLED | FS_OPT_SHDMEM_FUZZ); - if (write(fsrv->fsrv_ctl_fd, &send_status, 4) != 4) { + if (!be_quiet) { - FATAL("Writing to forkserver failed."); - - } + OKF("All right - new fork server model v%u is up.", version); - } + } - } else { + rlen = read(fsrv->fsrv_st_fd, &status, 4); - FATAL( - "Target requested sharedmem fuzzing, but we failed to enable " - "it."); + if (getenv("AFL_DEBUG")) { - } + ACTF("Forkserver options received: (0x%08x)", status); } - if ((status & FS_OPT_MAPSIZE) == FS_OPT_MAPSIZE) { + if ((status & FS_NEW_OPT_MAPSIZE)) { - u32 tmp_map_size = FS_OPT_GET_MAPSIZE(status); + u32 tmp_map_size; + rlen = read(fsrv->fsrv_st_fd, &tmp_map_size, 4); if (!fsrv->map_size) { fsrv->map_size = MAP_SIZE; } @@ -1059,7 +1098,8 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, FATAL( "Target's coverage map size of %u is larger than the one this " - "AFL++ is set with (%u). Either set AFL_MAP_SIZE=%u and restart " + "AFL++ is set with (%u). Either set AFL_MAP_SIZE=%u and " + "restart " " afl-fuzz, or change MAP_SIZE_POW2 in config.h and recompile " "afl-fuzz", tmp_map_size, fsrv->map_size, tmp_map_size); @@ -1068,106 +1108,326 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, fsrv->map_size = tmp_map_size; + } else { + + fsrv->real_map_size = fsrv->map_size = MAP_SIZE; + } - if ((status & FS_OPT_AUTODICT) == FS_OPT_AUTODICT) { + if (status & FS_NEW_OPT_SHDMEM_FUZZ) { + + if (fsrv->support_shmem_fuzz) { - if (!ignore_autodict) { + fsrv->use_shmem_fuzz = 1; + if (!be_quiet) { ACTF("Using SHARED MEMORY FUZZING feature."); } - if (fsrv->add_extra_func == NULL || fsrv->afl_ptr == NULL) { + } else { - // this is not afl-fuzz - or it is cmplog - we deny and return - if (fsrv->use_shmem_fuzz) { + FATAL( + "Target requested sharedmem fuzzing, but we failed to enable " + "it."); - status = (FS_OPT_ENABLED | FS_OPT_SHDMEM_FUZZ); + } - } else { + } - status = (FS_OPT_ENABLED); + if (status & FS_NEW_OPT_AUTODICT) { - } + // even if we do not need the dictionary we have to read it - if (write(fsrv->fsrv_ctl_fd, &status, 4) != 4) { + u32 dict_size; + if (read(fsrv->fsrv_st_fd, &dict_size, 4) != 4) { - FATAL("Writing to forkserver failed."); + FATAL("Reading from forkserver failed."); - } + } - return; + if (dict_size < 2 || dict_size > 0xffffff) { - } + FATAL("Dictionary has an illegal size: %d", dict_size); + + } + + u32 offset = 0, count = 0; + u8 *dict = ck_alloc(dict_size); + if (dict == NULL) { + + FATAL("Could not allocate %u bytes of autodictionary memory", + dict_size); + + } - if (!be_quiet) { ACTF("Using AUTODICT feature."); } + while (offset < dict_size) { - if (fsrv->use_shmem_fuzz) { + rlen = read(fsrv->fsrv_st_fd, dict + offset, dict_size - offset); + if (rlen > 0) { - status = (FS_OPT_ENABLED | FS_OPT_AUTODICT | FS_OPT_SHDMEM_FUZZ); + offset += rlen; } else { - status = (FS_OPT_ENABLED | FS_OPT_AUTODICT); + FATAL( + "Reading autodictionary fail at position %u with %u bytes " + "left.", + offset, dict_size - offset); + + } + + } + + offset = 0; + while (offset < dict_size && (u8)dict[offset] + offset < dict_size) { + + if (!ignore_autodict && fsrv->add_extra_func) { + + fsrv->add_extra_func(fsrv->afl_ptr, dict + offset + 1, + (u8)dict[offset]); + count++; } - if (write(fsrv->fsrv_ctl_fd, &status, 4) != 4) { + offset += (1 + dict[offset]); + + } + + if (!be_quiet && count) { + + ACTF("Loaded %u autodictionary entries", count); + + } + + ck_free(dict); + + } + + u32 status2; + rlen = read(fsrv->fsrv_st_fd, &status2, 4); + + if (status2 != keep) { + + FATAL("Error in forkserver communication (%08x=>%08x)", keep, status2); + + } + + } else { + + if (!fsrv->qemu_mode && !fsrv->cs_mode +#ifdef __linux__ + && !fsrv->nyx_mode +#endif + ) { + + WARNF( + "Old fork server model is used by the target, this still works " + "though."); + + } + + if (!be_quiet) { OKF("All right - old fork server is up."); } + + if (getenv("AFL_DEBUG")) { + + ACTF("Extended forkserver functions received (%08x).", status); + + } + + if ((status & FS_OPT_ERROR) == FS_OPT_ERROR) + report_error_and_exit(FS_OPT_GET_ERROR(status)); + + if (fsrv->cmplog_binary && !fsrv->qemu_mode) { + + FATAL("Target was compiled with outdated CMPLOG, recompile it!\n"); + + } + + if ((status & FS_OPT_ENABLED) == FS_OPT_ENABLED) { + + // workaround for recent AFL++ versions + if ((status & FS_OPT_OLD_AFLPP_WORKAROUND) == + FS_OPT_OLD_AFLPP_WORKAROUND) + status = (status & 0xf0ffffff); + + if ((status & FS_OPT_NEWCMPLOG) == 0 && fsrv->cmplog_binary) { + + if (fsrv->qemu_mode || fsrv->frida_mode) { + + report_error_and_exit(FS_ERROR_OLD_CMPLOG_QEMU); - FATAL("Writing to forkserver failed."); + } else { + + report_error_and_exit(FS_ERROR_OLD_CMPLOG); } - if (read(fsrv->fsrv_st_fd, &status, 4) != 4) { + } + + if ((status & FS_OPT_SNAPSHOT) == FS_OPT_SNAPSHOT) { + + fsrv->snapshot = 1; + if (!be_quiet) { ACTF("Using SNAPSHOT feature."); } + + } + + if ((status & FS_OPT_SHDMEM_FUZZ) == FS_OPT_SHDMEM_FUZZ) { + + if (fsrv->support_shmem_fuzz) { + + fsrv->use_shmem_fuzz = 1; + if (!be_quiet) { ACTF("Using SHARED MEMORY FUZZING feature."); } + + if ((status & FS_OPT_AUTODICT) == 0 || ignore_autodict) { + + u32 send_status = (FS_OPT_ENABLED | FS_OPT_SHDMEM_FUZZ); + if (write(fsrv->fsrv_ctl_fd, &send_status, 4) != 4) { + + FATAL("Writing to forkserver failed."); + + } + + } + + } else { - FATAL("Reading from forkserver failed."); + FATAL( + "Target requested sharedmem fuzzing, but we failed to enable " + "it."); } - if (status < 2 || (u32)status > 0xffffff) { + } + + if ((status & FS_OPT_MAPSIZE) == FS_OPT_MAPSIZE) { + + u32 tmp_map_size = FS_OPT_GET_MAPSIZE(status); + + if (!fsrv->map_size) { fsrv->map_size = MAP_SIZE; } + + fsrv->real_map_size = tmp_map_size; + + if (tmp_map_size % 64) { - FATAL("Dictionary has an illegal size: %d", status); + tmp_map_size = (((tmp_map_size + 63) >> 6) << 6); } - u32 offset = 0, count = 0; - u32 len = status; - u8 *dict = ck_alloc(len); - if (dict == NULL) { + if (!be_quiet) { ACTF("Target map size: %u", fsrv->real_map_size); } + if (tmp_map_size > fsrv->map_size) { - FATAL("Could not allocate %u bytes of autodictionary memory", len); + FATAL( + "Target's coverage map size of %u is larger than the one this " + "AFL++ is set with (%u). Either set AFL_MAP_SIZE=%u and " + "restart " + " afl-fuzz, or change MAP_SIZE_POW2 in config.h and recompile " + "afl-fuzz", + tmp_map_size, fsrv->map_size, tmp_map_size); } - while (len != 0) { + fsrv->map_size = tmp_map_size; - rlen = read(fsrv->fsrv_st_fd, dict + offset, len); - if (rlen > 0) { + } - len -= rlen; - offset += rlen; + if ((status & FS_OPT_AUTODICT) == FS_OPT_AUTODICT) { + + if (!ignore_autodict) { + + if (fsrv->add_extra_func == NULL || fsrv->afl_ptr == NULL) { + + // this is not afl-fuzz - or it is cmplog - we deny and return + if (fsrv->use_shmem_fuzz) { + + status = (FS_OPT_ENABLED | FS_OPT_SHDMEM_FUZZ); + + } else { + + status = (FS_OPT_ENABLED); + + } + + if (write(fsrv->fsrv_ctl_fd, &status, 4) != 4) { + + FATAL("Writing to forkserver failed."); + + } + + return; + + } + + if (!be_quiet) { ACTF("Using AUTODICT feature."); } + + if (fsrv->use_shmem_fuzz) { + + status = (FS_OPT_ENABLED | FS_OPT_AUTODICT | FS_OPT_SHDMEM_FUZZ); } else { - FATAL( - "Reading autodictionary fail at position %u with %u bytes " - "left.", - offset, len); + status = (FS_OPT_ENABLED | FS_OPT_AUTODICT); } - } + if (write(fsrv->fsrv_ctl_fd, &status, 4) != 4) { - offset = 0; - while (offset < (u32)status && - (u8)dict[offset] + offset < (u32)status) { + FATAL("Writing to forkserver failed."); - fsrv->add_extra_func(fsrv->afl_ptr, dict + offset + 1, - (u8)dict[offset]); - offset += (1 + dict[offset]); - count++; + } - } + if (read(fsrv->fsrv_st_fd, &status, 4) != 4) { - if (!be_quiet) { ACTF("Loaded %u autodictionary entries", count); } - ck_free(dict); + FATAL("Reading from forkserver failed."); + + } + + if (status < 2 || (u32)status > 0xffffff) { + + FATAL("Dictionary has an illegal size: %d", status); + + } + + u32 offset = 0, count = 0; + u32 len = status; + u8 *dict = ck_alloc(len); + if (dict == NULL) { + + FATAL("Could not allocate %u bytes of autodictionary memory", + len); + + } + + while (len != 0) { + + rlen = read(fsrv->fsrv_st_fd, dict + offset, len); + if (rlen > 0) { + + len -= rlen; + offset += rlen; + + } else { + + FATAL( + "Reading autodictionary fail at position %u with %u bytes " + "left.", + offset, len); + + } + + } + + offset = 0; + while (offset < (u32)status && + (u8)dict[offset] + offset < (u32)status) { + + fsrv->add_extra_func(fsrv->afl_ptr, dict + offset + 1, + (u8)dict[offset]); + offset += (1 + dict[offset]); + count++; + + } + + if (!be_quiet) { ACTF("Loaded %u autodictionary entries", count); } + ck_free(dict); + + } } @@ -1226,7 +1486,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, " - Less likely, there is a horrible bug in the fuzzer. If other " "options\n" - " fail, poke <afl-users@googlegroups.com> for troubleshooting " + " fail, poke the Awesome Fuzzing Discord for troubleshooting " "tips.\n"); } else { @@ -1271,7 +1531,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, " - Less likely, there is a horrible bug in the fuzzer. If other " "options\n" - " fail, poke <afl-users@googlegroups.com> for troubleshooting " + " fail, poke the Awesome Fuzzing Discord for troubleshooting " "tips.\n", stringify_mem_size(val_buf, sizeof(val_buf), fsrv->mem_limit << 20), fsrv->mem_limit - 1); @@ -1321,7 +1581,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, " Retry with setting AFL_MAP_SIZE=10000000.\n\n" "Otherwise there is a horrible bug in the fuzzer.\n" - "Poke <afl-users@googlegroups.com> for troubleshooting tips.\n"); + "Poke the Awesome Fuzzing Discord for troubleshooting tips.\n"); } else { @@ -1370,7 +1630,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, " - Less likely, there is a horrible bug in the fuzzer. If other " "options\n" - " fail, poke <afl-users@googlegroups.com> for troubleshooting " + " fail, poke the Awesome Fuzzing Discord for troubleshooting " "tips.\n", getenv(DEFER_ENV_VAR) ? " - You are using deferred forkserver, but __AFL_INIT() is " @@ -1548,6 +1808,11 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, u32 exec_ms; u32 write_value = fsrv->last_run_timed_out; +#ifdef AFL_PERSISTENT_RECORD + fsrv_run_result_t retval = FSRV_RUN_OK; + char *persistent_out_fmt; +#endif + #ifdef __linux__ if (fsrv->nyx_mode) { @@ -1577,11 +1842,13 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, case Timeout: return FSRV_RUN_TMOUT; case InvalidWriteToPayload: + if (!!getenv("AFL_NYX_HANDLE_INVALID_WRITE")) { return FSRV_RUN_CRASH; } + /* ??? */ FATAL("FixMe: Nyx InvalidWriteToPayload handler is missing"); break; case Abort: - FATAL("Error: Nyx abort occured..."); + FATAL("Error: Nyx abort occurred..."); case IoError: if (*stop_soon_p) { @@ -1595,7 +1862,7 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, break; case Error: - FATAL("Error: Nyx runtime error has occured..."); + FATAL("Error: Nyx runtime error has occurred..."); break; } @@ -1610,7 +1877,7 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, territory. */ #ifdef __linux__ - if (!fsrv->nyx_mode) { + if (likely(!fsrv->nyx_mode)) { memset(fsrv->trace_bits, 0, fsrv->map_size); MEM_BARRIER(); @@ -1680,7 +1947,7 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, if (exec_ms > timeout) { - /* If there was no response from forkserver after timeout seconds, + /* If there was no response from forkserver after timeout milliseconds, we kill the child. The forkserver should inform us afterwards */ s32 tmp_pid = fsrv->child_pid; @@ -1747,6 +2014,18 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, if (unlikely(fsrv->last_run_timed_out)) { fsrv->last_kill_signal = fsrv->child_kill_signal; + +#ifdef AFL_PERSISTENT_RECORD + if (unlikely(fsrv->persistent_record)) { + + retval = FSRV_RUN_TMOUT; + persistent_out_fmt = "%s/hangs/RECORD:%06u,cnt:%06u%s%s"; + goto store_persistent_record; + + } + +#endif + return FSRV_RUN_TMOUT; } @@ -1768,48 +2047,66 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, (fsrv->uses_crash_exitcode && WEXITSTATUS(fsrv->child_status) == fsrv->crash_exitcode))) { + /* For a proper crash, set last_kill_signal to WTERMSIG, else set it to 0 */ + fsrv->last_kill_signal = + WIFSIGNALED(fsrv->child_status) ? WTERMSIG(fsrv->child_status) : 0; + #ifdef AFL_PERSISTENT_RECORD if (unlikely(fsrv->persistent_record)) { - char fn[PATH_MAX]; - u32 i, writecnt = 0; - for (i = 0; i < fsrv->persistent_record; ++i) { + retval = FSRV_RUN_CRASH; + persistent_out_fmt = "%s/crashes/RECORD:%06u,cnt:%06u%s%s"; + goto store_persistent_record; - u32 entry = (i + fsrv->persistent_record_idx) % fsrv->persistent_record; - u8 *data = fsrv->persistent_record_data[entry]; - u32 len = fsrv->persistent_record_len[entry]; - if (likely(len && data)) { + } - snprintf(fn, sizeof(fn), "%s/RECORD:%06u,cnt:%06u", - fsrv->persistent_record_dir, fsrv->persistent_record_cnt, - writecnt++); - int fd = open(fn, O_CREAT | O_TRUNC | O_WRONLY, 0644); - if (fd >= 0) { +#endif - ck_write(fd, data, len, fn); - close(fd); + return FSRV_RUN_CRASH; - } + } - } + /* success :) */ + return FSRV_RUN_OK; - } +#ifdef AFL_PERSISTENT_RECORD +store_persistent_record: { + + char fn[PATH_MAX]; + u32 i, writecnt = 0; + for (i = 0; i < fsrv->persistent_record; ++i) { + + u32 entry = (i + fsrv->persistent_record_idx) % fsrv->persistent_record; + u8 *data = fsrv->persistent_record_data[entry]; + u32 len = fsrv->persistent_record_len[entry]; + if (likely(len && data)) { + + snprintf( + fn, sizeof(fn), persistent_out_fmt, fsrv->persistent_record_dir, + fsrv->persistent_record_cnt, writecnt++, + ((afl_state_t *)(fsrv->afl_ptr))->file_extension ? "." : "", + ((afl_state_t *)(fsrv->afl_ptr))->file_extension + ? (const char *)((afl_state_t *)(fsrv->afl_ptr))->file_extension + : ""); + int fd = open(fn, O_CREAT | O_TRUNC | O_WRONLY, 0644); + if (fd >= 0) { + + ck_write(fd, data, len, fn); + close(fd); - ++fsrv->persistent_record_cnt; + } } -#endif + } - /* For a proper crash, set last_kill_signal to WTERMSIG, else set it to 0 */ - fsrv->last_kill_signal = - WIFSIGNALED(fsrv->child_status) ? WTERMSIG(fsrv->child_status) : 0; - return FSRV_RUN_CRASH; + ++fsrv->persistent_record_cnt; - } + return retval; - /* success :) */ - return FSRV_RUN_OK; +} + +#endif } diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 556bb5d1..03bc5d6c 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -459,6 +459,17 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { if (unlikely(fault == FSRV_RUN_TMOUT && afl->afl_env.afl_ignore_timeouts)) { + if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) { + + classify_counts(&afl->fsrv); + u64 cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); + + // Saturated increment + if (likely(afl->n_fuzz[cksum % N_FUZZ_SIZE] < 0xFFFFFFFF)) + afl->n_fuzz[cksum % N_FUZZ_SIZE]++; + + } + return 0; } @@ -474,7 +485,7 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { /* Generating a hash on every input is super expensive. Bad idea and should only be used for special schedules */ - if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) { + if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) { classify_counts(&afl->fsrv); classified = 1; @@ -516,23 +527,56 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { #ifndef SIMPLE_FILES - queue_fn = - alloc_printf("%s/queue/id:%06u,%s", afl->out_dir, afl->queued_items, - describe_op(afl, new_bits + is_timeout, - NAME_MAX - strlen("id:000000,"))); + if (!afl->afl_env.afl_sha1_filenames) { + + queue_fn = alloc_printf( + "%s/queue/id:%06u,%s%s%s", afl->out_dir, afl->queued_items, + describe_op(afl, new_bits + is_timeout, + NAME_MAX - strlen("id:000000,")), + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + + } else { + + const char *hex = sha1_hex(mem, len); + queue_fn = alloc_printf( + "%s/queue/%s%s%s", afl->out_dir, hex, afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + ck_free((char *)hex); + + } #else - queue_fn = - alloc_printf("%s/queue/id_%06u", afl->out_dir, afl->queued_items); + queue_fn = alloc_printf( + "%s/queue/id_%06u", afl->out_dir, afl->queued_items, + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); #endif /* ^!SIMPLE_FILES */ - fd = open(queue_fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); - if (unlikely(fd < 0)) { PFATAL("Unable to create '%s'", queue_fn); } - ck_write(fd, mem, len, queue_fn); - close(fd); + fd = permissive_create(afl, queue_fn); + if (likely(fd >= 0)) { + + ck_write(fd, mem, len, queue_fn); + close(fd); + + } + add_to_queue(afl, queue_fn, len, 0); + if (unlikely(afl->fuzz_mode) && + likely(afl->switch_fuzz_mode && !afl->non_instrumented_mode)) { + + if (afl->afl_env.afl_no_ui) { + + ACTF("New coverage found, switching back to exploration mode."); + + } + + afl->fuzz_mode = 0; + + } + #ifdef INTROSPECTION if (afl->custom_mutators_count && afl->current_custom_fuzz) { @@ -715,14 +759,29 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { #ifndef SIMPLE_FILES - snprintf(fn, PATH_MAX, "%s/hangs/id:%06llu,%s", afl->out_dir, - afl->saved_hangs, - describe_op(afl, 0, NAME_MAX - strlen("id:000000,"))); + if (!afl->afl_env.afl_sha1_filenames) { + + snprintf(fn, PATH_MAX, "%s/hangs/id:%06llu,%s%s%s", afl->out_dir, + afl->saved_hangs, + describe_op(afl, 0, NAME_MAX - strlen("id:000000,")), + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + + } else { + + const char *hex = sha1_hex(mem, len); + snprintf(fn, PATH_MAX, "%s/hangs/%s%s%s", afl->out_dir, hex, + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + ck_free((char *)hex); + + } #else - snprintf(fn, PATH_MAX, "%s/hangs/id_%06llu", afl->out_dir, - afl->saved_hangs); + snprintf(fn, PATH_MAX, "%s/hangs/id_%06llu%s%s", afl->out_dir, + afl->saved_hangs, afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); #endif /* ^!SIMPLE_FILES */ @@ -768,14 +827,30 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { #ifndef SIMPLE_FILES - snprintf(fn, PATH_MAX, "%s/crashes/id:%06llu,sig:%02u,%s", afl->out_dir, - afl->saved_crashes, afl->fsrv.last_kill_signal, - describe_op(afl, 0, NAME_MAX - strlen("id:000000,sig:00,"))); + if (!afl->afl_env.afl_sha1_filenames) { + + snprintf(fn, PATH_MAX, "%s/crashes/id:%06llu,sig:%02u,%s%s%s", + afl->out_dir, afl->saved_crashes, afl->fsrv.last_kill_signal, + describe_op(afl, 0, NAME_MAX - strlen("id:000000,sig:00,")), + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + + } else { + + const char *hex = sha1_hex(mem, len); + snprintf(fn, PATH_MAX, "%s/crashes/%s%s%s", afl->out_dir, hex, + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + ck_free((char *)hex); + + } #else - snprintf(fn, PATH_MAX, "%s/crashes/id_%06llu_%02u", afl->out_dir, - afl->saved_crashes, afl->fsrv.last_kill_signal); + snprintf(fn, PATH_MAX, "%s/crashes/id_%06llu_%02u%s%s", afl->out_dir, + afl->saved_crashes, afl->fsrv.last_kill_signal, + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); #endif /* ^!SIMPLE_FILES */ @@ -838,10 +913,13 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { /* If we're here, we apparently want to save the crash or hang test case, too. */ - fd = open(fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); - if (unlikely(fd < 0)) { PFATAL("Unable to create '%s'", fn); } - ck_write(fd, mem, len, fn); - close(fd); + fd = permissive_create(afl, fn); + if (fd >= 0) { + + ck_write(fd, mem, len, fn); + close(fd); + + } #ifdef __linux__ if (afl->fsrv.nyx_mode && fault == FSRV_RUN_CRASH) { @@ -853,7 +931,8 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { if (unlikely(fd < 0)) { PFATAL("Unable to create '%s'", fn_log); } u32 nyx_aux_string_len = afl->fsrv.nyx_handlers->nyx_get_aux_string( - afl->fsrv.nyx_runner, afl->fsrv.nyx_aux_string, 0x1000); + afl->fsrv.nyx_runner, afl->fsrv.nyx_aux_string, + afl->fsrv.nyx_aux_string_len); ck_write(fd, afl->fsrv.nyx_aux_string, nyx_aux_string_len, fn_log); close(fd); diff --git a/src/afl-fuzz-cmplog.c b/src/afl-fuzz-cmplog.c index 3e6432ca..8c48eb49 100644 --- a/src/afl-fuzz-cmplog.c +++ b/src/afl-fuzz-cmplog.c @@ -7,11 +7,11 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-extras.c b/src/afl-fuzz-extras.c index f6de11ae..55b6be04 100644 --- a/src/afl-fuzz-extras.c +++ b/src/afl-fuzz-extras.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -176,6 +176,8 @@ void load_extras_file(afl_state_t *afl, u8 *fname, u32 *min_len, u32 *max_len, afl->extras = afl_realloc((void **)&afl->extras, (afl->extras_cnt + 1) * sizeof(struct extra_data)); + char *hexdigits = "0123456789abcdef"; + if (unlikely(!afl->extras)) { PFATAL("alloc"); } wptr = afl->extras[afl->extras_cnt].data = ck_alloc(rptr - lptr); @@ -184,13 +186,12 @@ void load_extras_file(afl_state_t *afl, u8 *fname, u32 *min_len, u32 *max_len, while (*lptr) { - char *hexdigits = "0123456789abcdef"; - switch (*lptr) { case 1 ... 31: case 128 ... 255: WARNF("Non-printable characters in line %u.", cur_line); + ++lptr; continue; break; @@ -741,8 +742,11 @@ void save_auto(afl_state_t *afl) { for (i = 0; i < MIN((u32)USE_AUTO_EXTRAS, afl->a_extras_cnt); ++i) { - u8 *fn = - alloc_printf("%s/queue/.state/auto_extras/auto_%06u", afl->out_dir, i); + u8 *fn = alloc_printf( + "%s/queue/.state/auto_extras/auto_%06u%s%s", afl->out_dir, i, + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + s32 fd; fd = open(fn, O_WRONLY | O_CREAT | O_TRUNC, DEFAULT_PERMISSION); diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index baf56a5f..7310e49f 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -124,6 +124,9 @@ void bind_to_free_cpu(afl_state_t *afl) { } WARNF("Not binding to a CPU core (AFL_NO_AFFINITY set)."); + #ifdef __linux__ + if (afl->fsrv.nyx_mode) { afl->fsrv.nyx_bind_cpu_id = 0; } + #endif return; } @@ -151,6 +154,9 @@ void bind_to_free_cpu(afl_state_t *afl) { } else { OKF("CPU binding request using -b %d successful.", afl->cpu_to_bind); + #ifdef __linux__ + if (afl->fsrv.nyx_mode) { afl->fsrv.nyx_bind_cpu_id = afl->cpu_to_bind; } + #endif } @@ -453,6 +459,24 @@ void bind_to_free_cpu(afl_state_t *afl) { #endif /* HAVE_AFFINITY */ +/* transforms spaces in a string to underscores (inplace) */ + +static void no_spaces(u8 *string) { + + if (string) { + + u8 *ptr = string; + while (*ptr != 0) { + + if (*ptr == ' ') { *ptr = '_'; } + ++ptr; + + } + + } + +} + /* Shuffle an array of pointers. Might be slightly biased. */ static void shuffle_ptrs(afl_state_t *afl, void **ptrs, u32 cnt) { @@ -553,6 +577,8 @@ void read_foreign_testcases(afl_state_t *afl, int first) { afl->stage_cur = 0; afl->stage_max = 0; + show_stats(afl); + for (i = 0; i < (u32)nl_cnt; ++i) { struct stat st; @@ -631,7 +657,12 @@ void read_foreign_testcases(afl_state_t *afl, int first) { munmap(mem, st.st_size); close(fd); - if (st.st_mtime > mtime_max) mtime_max = st.st_mtime; + if (st.st_mtime > mtime_max) { + + mtime_max = st.st_mtime; + show_stats(afl); + + } } @@ -908,6 +939,14 @@ void perform_dry_run(afl_state_t *afl) { res = calibrate_case(afl, q, use_mem, 0, 1); + /* For AFLFast schedules we update the queue entry */ + if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE) && + likely(q->exec_cksum)) { + + q->n_fuzz_entry = q->exec_cksum % N_FUZZ_SIZE; + + } + if (afl->stop_soon) { return; } if (res == afl->crash_mode || res == FSRV_RUN_NOBITS) { @@ -942,6 +981,7 @@ void perform_dry_run(afl_state_t *afl) { if (!q->was_fuzzed) { q->was_fuzzed = 1; + afl->reinit_table = 1; --afl->pending_not_fuzzed; --afl->active_items; @@ -951,19 +991,48 @@ void perform_dry_run(afl_state_t *afl) { } else { - SAYF("\n" cLRD "[-] " cRST - "The program took more than %u ms to process one of the initial " - "test cases.\n" - " This is bad news; raising the limit with the -t option is " - "possible, but\n" - " will probably make the fuzzing process extremely slow.\n\n" + static int say_once = 0; + + if (!say_once) { + + SAYF( + "\n" cLRD "[-] " cRST + "The program took more than %u ms to process one of the " + "initial " + "test cases.\n" + " This is bad news; raising the limit with the -t option is " + "possible, but\n" + " will probably make the fuzzing process extremely slow.\n\n" + + " If this test case is just a fluke, the other option is to " + "just avoid it\n" + " altogether, and find one that is less of a CPU hog.\n", + afl->fsrv.exec_tmout); + + if (!afl->afl_env.afl_ignore_seed_problems) { + + FATAL("Test case '%s' results in a timeout", fn); - " If this test case is just a fluke, the other option is to " - "just avoid it\n" - " altogether, and find one that is less of a CPU hog.\n", - afl->fsrv.exec_tmout); + } - FATAL("Test case '%s' results in a timeout", fn); + say_once = 1; + + } + + if (!q->was_fuzzed) { + + q->was_fuzzed = 1; + afl->reinit_table = 1; + --afl->pending_not_fuzzed; + --afl->active_items; + + } + + q->disabled = 1; + q->perf_score = 0; + + WARNF("Test case '%s' results in a timeout, skipping", fn); + break; } @@ -1012,7 +1081,7 @@ void perform_dry_run(afl_state_t *afl) { " - Least likely, there is a horrible bug in the fuzzer. If " "other options\n" - " fail, poke <afl-users@googlegroups.com> for " + " fail, poke the Awesome Fuzzing Discord for " "troubleshooting tips.\n", stringify_mem_size(val_buf, sizeof(val_buf), afl->fsrv.mem_limit << 20), @@ -1041,7 +1110,7 @@ void perform_dry_run(afl_state_t *afl) { " - Least likely, there is a horrible bug in the fuzzer. If " "other options\n" - " fail, poke <afl-users@googlegroups.com> for " + " fail, poke the Awesome Fuzzing Discord for " "troubleshooting tips.\n"); } @@ -1058,7 +1127,19 @@ void perform_dry_run(afl_state_t *afl) { } else { - WARNF("Test case '%s' results in a crash, skipping", fn); + if (afl->afl_env.afl_crashing_seeds_as_new_crash) { + + WARNF( + "Test case '%s' results in a crash, " + "as AFL_CRASHING_SEEDS_AS_NEW_CRASH is set, " + "saving as a new crash", + fn); + + } else { + + WARNF("Test case '%s' results in a crash, skipping", fn); + + } } @@ -1073,41 +1154,118 @@ void perform_dry_run(afl_state_t *afl) { if (!q->was_fuzzed) { q->was_fuzzed = 1; + afl->reinit_table = 1; --afl->pending_not_fuzzed; --afl->active_items; } - q->disabled = 1; - q->perf_score = 0; + /* Crashing seeds will be regarded as new crashes on startup */ + if (afl->afl_env.afl_crashing_seeds_as_new_crash) { - u32 i = 0; - while (unlikely(i < afl->queued_items && afl->queue_buf[i] && - afl->queue_buf[i]->disabled)) { + ++afl->total_crashes; - ++i; + if (likely(!afl->non_instrumented_mode)) { - } + classify_counts(&afl->fsrv); + + simplify_trace(afl, afl->fsrv.trace_bits); + + if (!has_new_bits(afl, afl->virgin_crash)) { break; } + + } + + if (unlikely(!afl->saved_crashes) && + (afl->afl_env.afl_no_crash_readme != 1)) { - if (i < afl->queued_items && afl->queue_buf[i]) { + write_crash_readme(afl); - afl->queue = afl->queue_buf[i]; + } + + u8 crash_fn[PATH_MAX]; + u8 *use_name = strstr(q->fname, ",orig:"); + + afl->stage_name = "dry_run"; + afl->stage_short = "dry_run"; + +#ifndef SIMPLE_FILES + + if (!afl->afl_env.afl_sha1_filenames) { + + snprintf( + crash_fn, PATH_MAX, "%s/crashes/id:%06llu,sig:%02u,%s%s%s%s", + afl->out_dir, afl->saved_crashes, afl->fsrv.last_kill_signal, + describe_op( + afl, 0, + NAME_MAX - strlen("id:000000,sig:00,") - strlen(use_name)), + use_name, afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + + } else { + + const char *hex = sha1_hex(use_mem, read_len); + snprintf( + crash_fn, PATH_MAX, "%s/crashes/%s%s%s", afl->out_dir, hex, + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + ck_free((char *)hex); + + } + +#else + + snprintf( + crash_fn, PATH_MAX, "%s/crashes/id_%06llu_%02u%s%s", afl->out_dir, + afl->saved_crashes, afl->fsrv.last_kill_signal, + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + +#endif + + ++afl->saved_crashes; + + fd = open(crash_fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); + if (unlikely(fd < 0)) { PFATAL("Unable to create '%s'", crash_fn); } + ck_write(fd, use_mem, read_len, crash_fn); + close(fd); + + afl->last_crash_time = get_cur_time(); + afl->last_crash_execs = afl->fsrv.total_execs; } else { - afl->queue = afl->queue_buf[0]; + u32 i = 0; + while (unlikely(i < afl->queued_items && afl->queue_buf[i] && + afl->queue_buf[i]->disabled)) { - } + ++i; + + } + + if (i < afl->queued_items && afl->queue_buf[i]) { + + afl->queue = afl->queue_buf[i]; + + } else { + + afl->queue = afl->queue_buf[0]; + + } + + afl->max_depth = 0; + for (i = 0; i < afl->queued_items && likely(afl->queue_buf[i]); i++) { - afl->max_depth = 0; - for (i = 0; i < afl->queued_items && likely(afl->queue_buf[i]); i++) { + if (!afl->queue_buf[i]->disabled && + afl->queue_buf[i]->depth > afl->max_depth) + afl->max_depth = afl->queue_buf[i]->depth; - if (!afl->queue_buf[i]->disabled && - afl->queue_buf[i]->depth > afl->max_depth) - afl->max_depth = afl->queue_buf[i]->depth; + } } + q->disabled = 1; + q->perf_score = 0; + break; case FSRV_RUN_ERROR: @@ -1192,6 +1350,7 @@ void perform_dry_run(afl_state_t *afl) { if (!p->was_fuzzed) { p->was_fuzzed = 1; + afl->reinit_table = 1; --afl->pending_not_fuzzed; --afl->active_items; @@ -1212,6 +1371,7 @@ void perform_dry_run(afl_state_t *afl) { if (!q->was_fuzzed) { q->was_fuzzed = 1; + afl->reinit_table = 1; --afl->pending_not_fuzzed; --afl->active_items; @@ -1262,11 +1422,11 @@ void perform_dry_run(afl_state_t *afl) { static void link_or_copy(u8 *old_path, u8 *new_path) { s32 i = link(old_path, new_path); + if (!i) { return; } + s32 sfd, dfd; u8 *tmp; - if (!i) { return; } - sfd = open(old_path, O_RDONLY); if (sfd < 0) { PFATAL("Unable to open '%s'", old_path); } @@ -1329,7 +1489,9 @@ void pivot_inputs(afl_state_t *afl) { u32 src_id; afl->resuming_fuzz = 1; - nfn = alloc_printf("%s/queue/%s", afl->out_dir, rsl); + nfn = alloc_printf( + "%s/queue/%s%s%s", afl->out_dir, rsl, afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); /* Since we're at it, let's also get the parent and figure out the appropriate depth for this entry. */ @@ -1369,12 +1531,33 @@ void pivot_inputs(afl_state_t *afl) { } - nfn = alloc_printf("%s/queue/id:%06u,time:0,execs:%llu,orig:%s", - afl->out_dir, id, afl->fsrv.total_execs, use_name); + if (!afl->afl_env.afl_sha1_filenames) { + + nfn = alloc_printf( + "%s/queue/id:%06u,time:0,execs:%llu,orig:%s%s%s", afl->out_dir, id, + afl->fsrv.total_execs, use_name, afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + + } else { + + const char *hex = sha1_hex_for_file(q->fname, q->len); + nfn = alloc_printf( + "%s/queue/%s%s%s", afl->out_dir, hex, + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); + ck_free((char *)hex); + + } + + u8 *pos = strrchr(nfn, '/'); + no_spaces(pos + 30); #else - nfn = alloc_printf("%s/queue/id_%06u", afl->out_dir, id); + nfn = alloc_printf( + "%s/queue/id_%06u%s%s", afl->out_dir, id, + afl->file_extension ? "." : "", + afl->file_extension ? (const char *)afl->file_extension : ""); #endif /* ^!SIMPLE_FILES */ @@ -1542,8 +1725,8 @@ double get_runnable_processes(void) { processes well. */ FILE *f = fopen("/proc/stat", "r"); - u8 tmp[1024]; - u32 val = 0; + u8 tmp[1024]; + u32 val = 0; if (!f) { return 0; } @@ -1581,10 +1764,11 @@ double get_runnable_processes(void) { void nuke_resume_dir(afl_state_t *afl) { - u8 *fn; + u8 *const case_prefix = afl->afl_env.afl_sha1_filenames ? "" : CASE_PREFIX; + u8 *fn; fn = alloc_printf("%s/_resume/.state/deterministic_done", afl->out_dir); - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); fn = alloc_printf("%s/_resume/.state/auto_extras", afl->out_dir); @@ -1592,11 +1776,11 @@ void nuke_resume_dir(afl_state_t *afl) { ck_free(fn); fn = alloc_printf("%s/_resume/.state/redundant_edges", afl->out_dir); - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); fn = alloc_printf("%s/_resume/.state/variable_behavior", afl->out_dir); - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); fn = alloc_printf("%s/_resume/.state", afl->out_dir); @@ -1604,7 +1788,7 @@ void nuke_resume_dir(afl_state_t *afl) { ck_free(fn); fn = alloc_printf("%s/_resume", afl->out_dir); - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); return; @@ -1621,8 +1805,9 @@ dir_cleanup_failed: static void handle_existing_out_dir(afl_state_t *afl) { - FILE *f; - u8 *fn = alloc_printf("%s/fuzzer_stats", afl->out_dir); + u8 *const case_prefix = afl->afl_env.afl_sha1_filenames ? "" : CASE_PREFIX; + FILE *f; + u8 *fn = alloc_printf("%s/fuzzer_stats", afl->out_dir); /* See if the output directory is locked. If yes, bail out. If not, create a lock that will persist for the lifetime of the process @@ -1744,7 +1929,7 @@ static void handle_existing_out_dir(afl_state_t *afl) { /* Next, we need to clean up <afl->out_dir>/queue/.state/ subdirectories: */ fn = alloc_printf("%s/queue/.state/deterministic_done", afl->out_dir); - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); fn = alloc_printf("%s/queue/.state/auto_extras", afl->out_dir); @@ -1752,11 +1937,11 @@ static void handle_existing_out_dir(afl_state_t *afl) { ck_free(fn); fn = alloc_printf("%s/queue/.state/redundant_edges", afl->out_dir); - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); fn = alloc_printf("%s/queue/.state/variable_behavior", afl->out_dir); - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); /* Then, get rid of the .state subdirectory itself (should be empty by now) @@ -1767,7 +1952,7 @@ static void handle_existing_out_dir(afl_state_t *afl) { ck_free(fn); fn = alloc_printf("%s/queue", afl->out_dir); - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); /* All right, let's do <afl->out_dir>/crashes/id:* and @@ -1811,7 +1996,10 @@ static void handle_existing_out_dir(afl_state_t *afl) { } - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } +#ifdef AFL_PERSISTENT_RECORD + delete_files(fn, RECORD_PREFIX); +#endif + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); fn = alloc_printf("%s/hangs", afl->out_dir); @@ -1843,7 +2031,10 @@ static void handle_existing_out_dir(afl_state_t *afl) { } - if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } +#ifdef AFL_PERSISTENT_RECORD + delete_files(fn, RECORD_PREFIX); +#endif + if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; } ck_free(fn); /* And now, for some finishing touches. */ @@ -2126,6 +2317,21 @@ void setup_dirs_fds(afl_state_t *afl) { fflush(afl->fsrv.plot_file); +#ifdef INTROSPECTION + + tmp = alloc_printf("%s/plot_det_data", afl->out_dir); + + int fd = open(tmp, O_WRONLY | O_CREAT, DEFAULT_PERMISSION); + if (fd < 0) { PFATAL("Unable to create '%s'", tmp); } + ck_free(tmp); + + afl->fsrv.det_plot_file = fdopen(fd, "w"); + if (!afl->fsrv.det_plot_file) { PFATAL("fdopen() failed"); } + + if (afl->in_place_resume) { fseek(afl->fsrv.det_plot_file, 0, SEEK_END); } + +#endif + /* ignore errors */ } @@ -2199,7 +2405,8 @@ void check_crash_handling(void) { reporting the awful way. */ #if !TARGET_OS_IPHONE - if (system("launchctl list 2>/dev/null | grep -q '\\.ReportCrash$'")) return; + if (system("launchctl list 2>/dev/null | grep -q '\\.ReportCrash\\>'")) + return; SAYF( "\n" cLRD "[-] " cRST @@ -2226,7 +2433,7 @@ void check_crash_handling(void) { *BSD, so we can just let it slide for now. */ s32 fd = open("/proc/sys/kernel/core_pattern", O_RDONLY); - u8 fchar; + u8 fchar; if (fd < 0) { return; } @@ -2365,7 +2572,7 @@ void check_cpu_governor(afl_state_t *afl) { FATAL("Suboptimal CPU scaling governor"); #elif defined __APPLE__ - u64 min = 0, max = 0; + u64 min = 0, max = 0; size_t mlen = sizeof(min); if (afl->afl_env.afl_skip_cpufreq) return; diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 64dbe7c6..2f6af4bc 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -5,12 +5,12 @@ Originally written by Shengtuo Hu Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -397,6 +397,18 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { } + /* "afl_custom_post_run", optional */ + mutator->afl_custom_post_run = dlsym(dh, "afl_custom_post_run"); + if (!mutator->afl_custom_post_run) { + + ACTF("optional symbol 'afl_custom_post_run' not found."); + + } else { + + OKF("Found 'afl_custom_post_run'."); + + } + /* "afl_custom_queue_new_entry", optional */ mutator->afl_custom_queue_new_entry = dlsym(dh, "afl_custom_queue_new_entry"); if (!mutator->afl_custom_queue_new_entry) { diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index c6e9a295..74bb8cbc 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ #include <string.h> #include <limits.h> #include "cmplog.h" +#include "afl-mutations.h" /* MOpt */ @@ -70,50 +71,6 @@ static int select_algorithm(afl_state_t *afl, u32 max_algorithm) { } -/* Helper to choose random block len for block operations in fuzz_one(). - Doesn't return zero, provided that max_len is > 0. */ - -static inline u32 choose_block_len(afl_state_t *afl, u32 limit) { - - u32 min_value, max_value; - u32 rlim = MIN(afl->queue_cycle, (u32)3); - - if (unlikely(!afl->run_over10m)) { rlim = 1; } - - switch (rand_below(afl, rlim)) { - - case 0: - min_value = 1; - max_value = HAVOC_BLK_SMALL; - break; - - case 1: - min_value = HAVOC_BLK_SMALL; - max_value = HAVOC_BLK_MEDIUM; - break; - - default: - - if (likely(rand_below(afl, 10))) { - - min_value = HAVOC_BLK_MEDIUM; - max_value = HAVOC_BLK_LARGE; - - } else { - - min_value = HAVOC_BLK_LARGE; - max_value = HAVOC_BLK_XL; - - } - - } - - if (min_value >= limit) { min_value = 1; } - - return min_value + rand_below(afl, MIN(max_value, limit) - min_value + 1); - -} - /* Helper function to see if a particular change (xor_val = old ^ new) could be a product of deterministic bit flips with the lengths and stepovers attempted by afl-fuzz. This is used to avoid dupes in some of the @@ -372,9 +329,9 @@ u8 fuzz_one_original(afl_state_t *afl) { u32 len, temp_len; u32 j; u32 i; - u8 *in_buf, *out_buf, *orig_in, *ex_tmp, *eff_map = 0; + u8 *in_buf, *out_buf, *orig_in, *ex_tmp; u64 havoc_queued = 0, orig_hit_cnt, new_hit_cnt = 0, prev_cksum, _prev_cksum; - u32 splice_cycle = 0, perf_score = 100, orig_perf, eff_cnt = 1; + u32 splice_cycle = 0, perf_score = 100, orig_perf; u8 ret_val = 1, doing_det = 0; @@ -442,18 +399,24 @@ u8 fuzz_one_original(afl_state_t *afl) { #endif /* ^IGNORE_FINDS */ - if (unlikely(afl->not_on_tty)) { + if (likely(afl->not_on_tty)) { + + u8 time_tmp[64]; + u_simplestring_time_diff(time_tmp, afl->prev_run_time + get_cur_time(), + afl->start_time); ACTF( - "Fuzzing test case #%u (%u total, %llu crashes saved, " + "Fuzzing test case #%u (%u total, %llu crashes saved, state: %s, " + "mode=%s, " "perf_score=%0.0f, weight=%0.0f, favorite=%u, was_fuzzed=%u, " - "exec_us=%llu, hits=%u, map=%u, ascii=%u)...", + "exec_us=%llu, hits=%u, map=%u, ascii=%u, run_time=%s)...", afl->current_entry, afl->queued_items, afl->saved_crashes, + get_fuzzing_state(afl), afl->fuzz_mode ? "exploit" : "explore", afl->queue_cur->perf_score, afl->queue_cur->weight, afl->queue_cur->favored, afl->queue_cur->was_fuzzed, afl->queue_cur->exec_us, likely(afl->n_fuzz) ? afl->n_fuzz[afl->queue_cur->n_fuzz_entry] : 0, - afl->queue_cur->bitmap_size, afl->queue_cur->is_ascii); + afl->queue_cur->bitmap_size, afl->queue_cur->is_ascii, time_tmp); fflush(stdout); } @@ -582,12 +545,37 @@ u8 fuzz_one_original(afl_state_t *afl) { } + u64 before_det_time = get_cur_time(); +#ifdef INTROSPECTION + + u64 before_havoc_time; + u32 before_det_findings = afl->queued_items, + before_det_edges = count_non_255_bytes(afl, afl->virgin_bits), + before_havoc_findings, before_havoc_edges; + u8 is_logged = 0; + +#endif + if (!afl->skip_deterministic) { + + if (!skip_deterministic_stage(afl, in_buf, out_buf, len, before_det_time)) { + + goto abandon_entry; + + } + + } + + u8 *skip_eff_map = afl->queue_cur->skipdet_e->skip_eff_map; + /* Skip right away if -d is given, if it has not been chosen sufficiently often to warrant the expensive deterministic stage (fuzz_level), or if it has gone through deterministic testing in earlier, resumed runs (passed_det). */ + /* if skipdet decide to skip the seed or no interesting bytes found, + we skip the whole deterministic stage as well */ if (likely(afl->skip_deterministic) || likely(afl->queue_cur->passed_det) || + likely(!afl->queue_cur->skipdet_e->quick_eff_bytes) || likely(perf_score < (afl->queue_cur->depth * 30 <= afl->havoc_max_mult * 100 ? afl->queue_cur->depth * 30 @@ -614,13 +602,13 @@ u8 fuzz_one_original(afl_state_t *afl) { * SIMPLE BITFLIP (+dictionary construction) * *********************************************/ -#define FLIP_BIT(_ar, _b) \ - do { \ - \ - u8 *_arf = (u8 *)(_ar); \ - u32 _bf = (_b); \ - _arf[(_bf) >> 3] ^= (128 >> ((_bf)&7)); \ - \ +#define FLIP_BIT(_ar, _b) \ + do { \ + \ + u8 *_arf = (u8 *)(_ar); \ + u32 _bf = (_b); \ + _arf[(_bf) >> 3] ^= (128 >> ((_bf) & 7)); \ + \ } while (0) /* Single walking bit. */ @@ -646,6 +634,10 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->stage_cur_byte = afl->stage_cur >> 3; + if (!skip_eff_map[afl->stage_cur_byte]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + FLIP_BIT(out_buf, afl->stage_cur); #ifdef INTROSPECTION @@ -762,6 +754,10 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->stage_cur_byte = afl->stage_cur >> 3; + if (!skip_eff_map[afl->stage_cur_byte]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + FLIP_BIT(out_buf, afl->stage_cur); FLIP_BIT(out_buf, afl->stage_cur + 1); @@ -797,6 +793,10 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->stage_cur_byte = afl->stage_cur >> 3; + if (!skip_eff_map[afl->stage_cur_byte]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + FLIP_BIT(out_buf, afl->stage_cur); FLIP_BIT(out_buf, afl->stage_cur + 1); FLIP_BIT(out_buf, afl->stage_cur + 2); @@ -824,34 +824,6 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->queue_cur->stats_mutated += afl->stage_max; #endif - /* Effector map setup. These macros calculate: - - EFF_APOS - position of a particular file offset in the map. - EFF_ALEN - length of a map with a particular number of bytes. - EFF_SPAN_ALEN - map span for a sequence of bytes. - - */ - -#define EFF_APOS(_p) ((_p) >> EFF_MAP_SCALE2) -#define EFF_REM(_x) ((_x) & ((1 << EFF_MAP_SCALE2) - 1)) -#define EFF_ALEN(_l) (EFF_APOS(_l) + !!EFF_REM(_l)) -#define EFF_SPAN_ALEN(_p, _l) (EFF_APOS((_p) + (_l)-1) - EFF_APOS(_p) + 1) - - /* Initialize effector map for the next step (see comments below). Always - flag first and last byte as doing something. */ - - eff_map = afl_realloc(AFL_BUF_PARAM(eff), EFF_ALEN(len)); - if (unlikely(!eff_map)) { PFATAL("alloc"); } - memset(eff_map, 0, EFF_ALEN(len)); - eff_map[0] = 1; - - if (EFF_APOS(len - 1) != 0) { - - eff_map[EFF_APOS(len - 1)] = 1; - ++eff_cnt; - - } - /* Walking byte. */ afl->stage_name = "bitflip 8/8"; @@ -865,6 +837,10 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->stage_cur_byte = afl->stage_cur; + if (!skip_eff_map[afl->stage_cur_byte]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + out_buf[afl->stage_cur] ^= 0xFF; #ifdef INTROSPECTION @@ -874,59 +850,19 @@ u8 fuzz_one_original(afl_state_t *afl) { if (common_fuzz_stuff(afl, out_buf, len)) { goto abandon_entry; } - /* We also use this stage to pull off a simple trick: we identify - bytes that seem to have no effect on the current execution path - even when fully flipped - and we skip them during more expensive - deterministic stages, such as arithmetics or known ints. */ - - if (!eff_map[EFF_APOS(afl->stage_cur)]) { - - u64 cksum; - - /* If in non-instrumented mode or if the file is very short, just flag - everything without wasting time on checksums. */ - - if (!afl->non_instrumented_mode && len >= EFF_MIN_LEN) { - - cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); - - } else { - - cksum = ~prev_cksum; - - } - - if (cksum != prev_cksum) { - - eff_map[EFF_APOS(afl->stage_cur)] = 1; - ++eff_cnt; - - } - - } - out_buf[afl->stage_cur] ^= 0xFF; } - /* If the effector map is more than EFF_MAX_PERC dense, just flag the - whole thing as worth fuzzing, since we wouldn't be saving much time - anyway. */ + /* New effective bytes calculation. */ - if (eff_cnt != (u32)EFF_ALEN(len) && - eff_cnt * 100 / EFF_ALEN(len) > EFF_MAX_PERC) { + for (i = 0; i < len; i++) { - memset(eff_map, 1, EFF_ALEN(len)); - - afl->blocks_eff_select += EFF_ALEN(len); - - } else { - - afl->blocks_eff_select += eff_cnt; + if (skip_eff_map[i]) afl->blocks_eff_select += 1; } - afl->blocks_eff_total += EFF_ALEN(len); + afl->blocks_eff_total += len; new_hit_cnt = afl->queued_items + afl->saved_crashes; @@ -951,12 +887,9 @@ u8 fuzz_one_original(afl_state_t *afl) { /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)]) { + if (!skip_eff_map[i]) continue; - --afl->stage_max; - continue; - - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -996,13 +929,10 @@ u8 fuzz_one_original(afl_state_t *afl) { for (i = 0; i < len - 3; ++i) { /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)] && - !eff_map[EFF_APOS(i + 2)] && !eff_map[EFF_APOS(i + 3)]) { - --afl->stage_max; - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1053,12 +983,9 @@ skip_bitflip: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)]) { - - afl->stage_max -= 2 * ARITH_MAX; - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1140,12 +1067,9 @@ skip_bitflip: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)]) { - - afl->stage_max -= 4 * ARITH_MAX; - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1273,13 +1197,9 @@ skip_bitflip: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)] && - !eff_map[EFF_APOS(i + 2)] && !eff_map[EFF_APOS(i + 3)]) { + if (!skip_eff_map[i]) continue; - afl->stage_max -= 4 * ARITH_MAX; - continue; - - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1411,12 +1331,9 @@ skip_arith: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)]) { - - afl->stage_max -= sizeof(interesting_8); - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1474,12 +1391,9 @@ skip_arith: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)]) { + if (!skip_eff_map[i]) continue; - afl->stage_max -= sizeof(interesting_16); - continue; - - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1565,13 +1479,9 @@ skip_arith: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)] && - !eff_map[EFF_APOS(i + 2)] && !eff_map[EFF_APOS(i + 3)]) { - - afl->stage_max -= sizeof(interesting_32) >> 1; - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1663,6 +1573,10 @@ skip_interest: u32 last_len = 0; + if (!skip_eff_map[i]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + afl->stage_cur_byte = i; /* Extras are sorted by size, from smallest to largest. This means @@ -1680,9 +1594,7 @@ skip_interest: if ((afl->extras_cnt > afl->max_det_extras && rand_below(afl, afl->extras_cnt) >= afl->max_det_extras) || afl->extras[j].len > len - i || - !memcmp(afl->extras[j].data, out_buf + i, afl->extras[j].len) || - !memchr(eff_map + EFF_APOS(i), 1, - EFF_SPAN_ALEN(i, afl->extras[j].len))) { + !memcmp(afl->extras[j].data, out_buf + i, afl->extras[j].len)) { --afl->stage_max; continue; @@ -1730,6 +1642,10 @@ skip_interest: for (i = 0; i <= (u32)len; ++i) { + if (!skip_eff_map[i % len]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + afl->stage_cur_byte = i; for (j = 0; j < afl->extras_cnt; ++j) { @@ -1792,6 +1708,10 @@ skip_user_extras: u32 last_len = 0; + if (!skip_eff_map[i]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + afl->stage_cur_byte = i; u32 min_extra_len = MIN(afl->a_extras_cnt, (u32)USE_AUTO_EXTRAS); @@ -1800,9 +1720,7 @@ skip_user_extras: /* See the comment in the earlier code; extras are sorted by size. */ if (afl->a_extras[j].len > len - i || - !memcmp(afl->a_extras[j].data, out_buf + i, afl->a_extras[j].len) || - !memchr(eff_map + EFF_APOS(i), 1, - EFF_SPAN_ALEN(i, afl->a_extras[j].len))) { + !memcmp(afl->a_extras[j].data, out_buf + i, afl->a_extras[j].len)) { --afl->stage_max; continue; @@ -1850,6 +1768,10 @@ skip_user_extras: for (i = 0; i <= (u32)len; ++i) { + if (!skip_eff_map[i % len]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + afl->stage_cur_byte = i; for (j = 0; j < afl->a_extras_cnt; ++j) { @@ -1912,6 +1834,7 @@ custom_mutator_stage: afl->stage_name = "custom mutator"; afl->stage_short = "custom"; + afl->stage_cur = 0; afl->stage_val_type = STAGE_VAL_NONE; bool has_custom_fuzz = false; u32 shift = unlikely(afl->custom_only) ? 7 : 8; @@ -1931,6 +1854,8 @@ custom_mutator_stage: if (el->afl_custom_fuzz) { + havoc_queued = afl->queued_items; + afl->current_custom_fuzz = el; afl->stage_name = el->name_short; @@ -2054,6 +1979,19 @@ custom_mutator_stage: havoc_stage: +#ifdef INTROSPECTION + + if (!is_logged) { + + is_logged = 1; + before_havoc_findings = afl->queued_items; + before_havoc_edges = count_non_255_bytes(afl, afl->virgin_bits); + before_havoc_time = get_cur_time(); + + } + +#endif + if (unlikely(afl->custom_only)) { /* Force UI update */ @@ -2122,45 +2060,97 @@ havoc_stage: /* We essentially just do several thousand runs (depending on perf_score) where we take the input file and make random stacked tweaks. */ -#define MAX_HAVOC_ENTRY 64 -#define MUTATE_ASCII_DICT 64 + u32 *mutation_array; + u32 stack_max, rand_max; // stack_max_pow = afl->havoc_stack_pow2; + + switch (afl->input_mode) { + + case 1: { // TEXT - u32 r_max, r; + if (likely(afl->fuzz_mode == 0)) { // is exploration? + mutation_array = (unsigned int *)&binary_array; + rand_max = MUT_BIN_ARRAY_SIZE; - r_max = (MAX_HAVOC_ENTRY + 1) + (afl->extras_cnt ? 4 : 0) + - (afl->a_extras_cnt - ? (unlikely(afl->cmplog_binary && afl->queue_cur->is_ascii) - ? MUTATE_ASCII_DICT - : 4) - : 0); + } else { // exploitation mode - if (unlikely(afl->expand_havoc && afl->ready_for_splicing_count > 1)) { + mutation_array = (unsigned int *)&text_array; + rand_max = MUT_TXT_ARRAY_SIZE; - /* add expensive havoc cases here, they are activated after a full - cycle without finds happened */ + } + + break; + + } + + case 2: { // BINARY + + if (likely(afl->fuzz_mode == 0)) { // is exploration? + mutation_array = (unsigned int *)&mutation_strategy_exploration_binary; + rand_max = MUT_STRATEGY_ARRAY_SIZE; + + } else { // exploitation mode + + mutation_array = (unsigned int *)&mutation_strategy_exploitation_binary; + rand_max = MUT_STRATEGY_ARRAY_SIZE; + // or this one? we do not have enough binary bug benchmarks :-( + // mutation_array = (unsigned int *)&binary_array; + // rand_max = MUT_BIN_ARRAY_SIZE; + + } + + break; + + } + + default: { // DEFAULT/GENERIC - r_max += 4; + if (likely(afl->fuzz_mode == 0)) { // is exploration? + mutation_array = (unsigned int *)&binary_array; + rand_max = MUT_BIN_ARRAY_SIZE; + + } else { // exploitation mode + + mutation_array = (unsigned int *)&text_array; + rand_max = MUT_TXT_ARRAY_SIZE; + + } + + break; + + } } - if (unlikely(get_cur_time() - afl->last_find_time > 5000 /* 5 seconds */ && - afl->ready_for_splicing_count > 1)) { + /* + if (temp_len < 64) { + + --stack_max_pow; - /* add expensive havoc cases here if there is no findings in the last 5s */ + } else if (temp_len <= 8096) { - r_max += 4; + ++stack_max_pow; + + } else { + + ++stack_max_pow; } + */ + + stack_max = 1 << (1 + rand_below(afl, afl->havoc_stack_pow2)); + + // + (afl->extras_cnt ? 2 : 0) + (afl->a_extras_cnt ? 2 : 0); + for (afl->stage_cur = 0; afl->stage_cur < afl->stage_max; ++afl->stage_cur) { - u32 use_stacking = 1 << (1 + rand_below(afl, afl->havoc_stack_pow2)); + u32 use_stacking = 1 + rand_below(afl, stack_max); afl->stage_cur_val = use_stacking; #ifdef INTROSPECTION - snprintf(afl->mutation, sizeof(afl->mutation), "%s HAVOC-%u", - afl->queue_cur->fname, use_stacking); + snprintf(afl->mutation, sizeof(afl->mutation), "%s HAVOC-%u-%u", + afl->queue_cur->fname, afl->queue_cur->is_ascii, use_stacking); #endif for (i = 0; i < use_stacking; ++i) { @@ -2169,8 +2159,8 @@ havoc_stage: LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { - if (el->stacked_custom && - rand_below(afl, 100) < el->stacked_custom_prob) { + if (unlikely(el->stacked_custom && + rand_below(afl, 100) < el->stacked_custom_prob)) { u8 *custom_havoc_buf = NULL; size_t new_len = el->afl_custom_havoc_mutation( @@ -2200,159 +2190,173 @@ havoc_stage: } - switch ((r = rand_below(afl, r_max))) { + retry_havoc_step: { + + u32 r = rand_below(afl, rand_max), item; + + switch (mutation_array[r]) { - case 0 ... 3: { + case MUT_FLIPBIT: { /* Flip a single bit somewhere. Spooky! */ + u8 bit = rand_below(afl, 8); + u32 off = rand_below(afl, temp_len); + out_buf[off] ^= 1 << bit; #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " FLIP_BIT1"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " FLIP-BIT_%u", bit); strcat(afl->mutation, afl->m_tmp); #endif - FLIP_BIT(out_buf, rand_below(afl, temp_len << 3)); break; } - case 4 ... 7: { + case MUT_INTERESTING8: { /* Set byte to interesting value. */ + item = rand_below(afl, sizeof(interesting_8)); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING8"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING8_%u", item); strcat(afl->mutation, afl->m_tmp); #endif - out_buf[rand_below(afl, temp_len)] = - interesting_8[rand_below(afl, sizeof(interesting_8))]; + out_buf[rand_below(afl, temp_len)] = interesting_8[item]; break; } - case 8 ... 9: { + case MUT_INTERESTING16: { /* Set word to interesting value, little endian. */ - if (temp_len < 2) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry + item = rand_below(afl, sizeof(interesting_16) >> 1); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING16"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING16_%u", item); strcat(afl->mutation, afl->m_tmp); #endif + *(u16 *)(out_buf + rand_below(afl, temp_len - 1)) = - interesting_16[rand_below(afl, sizeof(interesting_16) >> 1)]; + interesting_16[item]; break; } - case 10 ... 11: { + case MUT_INTERESTING16BE: { /* Set word to interesting value, big endian. */ - if (temp_len < 2) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry + item = rand_below(afl, sizeof(interesting_16) >> 1); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING16BE"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING16BE_%u", item); strcat(afl->mutation, afl->m_tmp); #endif - *(u16 *)(out_buf + rand_below(afl, temp_len - 1)) = SWAP16( - interesting_16[rand_below(afl, sizeof(interesting_16) >> 1)]); + *(u16 *)(out_buf + rand_below(afl, temp_len - 1)) = + SWAP16(interesting_16[item]); break; } - case 12 ... 13: { + case MUT_INTERESTING32: { /* Set dword to interesting value, little endian. */ - if (temp_len < 4) { break; } + if (unlikely(temp_len < 4)) { break; } // no retry + item = rand_below(afl, sizeof(interesting_32) >> 2); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING32"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING32_%u", item); strcat(afl->mutation, afl->m_tmp); #endif + *(u32 *)(out_buf + rand_below(afl, temp_len - 3)) = - interesting_32[rand_below(afl, sizeof(interesting_32) >> 2)]; + interesting_32[item]; break; } - case 14 ... 15: { + case MUT_INTERESTING32BE: { /* Set dword to interesting value, big endian. */ - if (temp_len < 4) { break; } + if (unlikely(temp_len < 4)) { break; } // no retry + item = rand_below(afl, sizeof(interesting_32) >> 2); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING32BE"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INTERESTING32BE_%u", item); strcat(afl->mutation, afl->m_tmp); #endif - *(u32 *)(out_buf + rand_below(afl, temp_len - 3)) = SWAP32( - interesting_32[rand_below(afl, sizeof(interesting_32) >> 2)]); + *(u32 *)(out_buf + rand_below(afl, temp_len - 3)) = + SWAP32(interesting_32[item]); break; } - case 16 ... 19: { + case MUT_ARITH8_: { /* Randomly subtract from byte. */ + item = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH8_"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH8-_%u", item); strcat(afl->mutation, afl->m_tmp); #endif - out_buf[rand_below(afl, temp_len)] -= 1 + rand_below(afl, ARITH_MAX); + out_buf[rand_below(afl, temp_len)] -= item; break; } - case 20 ... 23: { + case MUT_ARITH8: { /* Randomly add to byte. */ + item = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH8+"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH8+_%u", item); strcat(afl->mutation, afl->m_tmp); #endif - out_buf[rand_below(afl, temp_len)] += 1 + rand_below(afl, ARITH_MAX); + out_buf[rand_below(afl, temp_len)] += item; break; } - case 24 ... 25: { + case MUT_ARITH16_: { /* Randomly subtract from word, little endian. */ - if (temp_len < 2) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry u32 pos = rand_below(afl, temp_len - 1); + item = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH16_-%u", pos); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH16-_%u", item); strcat(afl->mutation, afl->m_tmp); #endif - *(u16 *)(out_buf + pos) -= 1 + rand_below(afl, ARITH_MAX); + *(u16 *)(out_buf + pos) -= item; break; } - case 26 ... 27: { + case MUT_ARITH16BE_: { /* Randomly subtract from word, big endian. */ - if (temp_len < 2) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry u32 pos = rand_below(afl, temp_len - 1); u16 num = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH16_BE-%u_%u", pos, - num); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH16BE-_%u", num); strcat(afl->mutation, afl->m_tmp); #endif *(u16 *)(out_buf + pos) = @@ -2362,36 +2366,36 @@ havoc_stage: } - case 28 ... 29: { + case MUT_ARITH16: { /* Randomly add to word, little endian. */ - if (temp_len < 2) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry u32 pos = rand_below(afl, temp_len - 1); + item = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH16+-%u", pos); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH16+_%u", item); strcat(afl->mutation, afl->m_tmp); #endif - *(u16 *)(out_buf + pos) += 1 + rand_below(afl, ARITH_MAX); + *(u16 *)(out_buf + pos) += item; break; } - case 30 ... 31: { + case MUT_ARITH16BE: { /* Randomly add to word, big endian. */ - if (temp_len < 2) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry u32 pos = rand_below(afl, temp_len - 1); u16 num = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH16+BE-%u_%u", pos, - num); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH16BE+__%u", num); strcat(afl->mutation, afl->m_tmp); #endif *(u16 *)(out_buf + pos) = @@ -2401,36 +2405,36 @@ havoc_stage: } - case 32 ... 33: { + case MUT_ARITH32_: { /* Randomly subtract from dword, little endian. */ - if (temp_len < 4) { break; } + if (unlikely(temp_len < 4)) { break; } // no retry u32 pos = rand_below(afl, temp_len - 3); + item = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH32_-%u", pos); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH32-_%u", item); strcat(afl->mutation, afl->m_tmp); #endif - *(u32 *)(out_buf + pos) -= 1 + rand_below(afl, ARITH_MAX); + *(u32 *)(out_buf + pos) -= item; break; } - case 34 ... 35: { + case MUT_ARITH32BE_: { /* Randomly subtract from dword, big endian. */ - if (temp_len < 4) { break; } + if (unlikely(temp_len < 4)) { break; } // no retry u32 pos = rand_below(afl, temp_len - 3); u32 num = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH32_BE-%u-%u", pos, - num); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH32BE-_%u", num); strcat(afl->mutation, afl->m_tmp); #endif *(u32 *)(out_buf + pos) = @@ -2440,36 +2444,36 @@ havoc_stage: } - case 36 ... 37: { + case MUT_ARITH32: { /* Randomly add to dword, little endian. */ - if (temp_len < 4) { break; } + if (unlikely(temp_len < 4)) { break; } // no retry u32 pos = rand_below(afl, temp_len - 3); + item = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH32+-%u", pos); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH32+_%u", item); strcat(afl->mutation, afl->m_tmp); #endif - *(u32 *)(out_buf + pos) += 1 + rand_below(afl, ARITH_MAX); + *(u32 *)(out_buf + pos) += item; break; } - case 38 ... 39: { + case MUT_ARITH32BE: { /* Randomly add to dword, big endian. */ - if (temp_len < 4) { break; } + if (unlikely(temp_len < 4)) { break; } // no retry u32 pos = rand_below(afl, temp_len - 3); u32 num = 1 + rand_below(afl, ARITH_MAX); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH32+BE-%u-%u", pos, - num); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ARITH32BE+_%u", num); strcat(afl->mutation, afl->m_tmp); #endif *(u32 *)(out_buf + pos) = @@ -2479,24 +2483,27 @@ havoc_stage: } - case 40 ... 43: { + case MUT_RAND8: { /* Just set a random byte to a random value. Because, why not. We use XOR with 1-255 to eliminate the possibility of a no-op. */ + u32 pos = rand_below(afl, temp_len); + item = 1 + rand_below(afl, 255); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " RAND8"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " RAND8_%u", + out_buf[pos] ^ item); strcat(afl->mutation, afl->m_tmp); #endif - out_buf[rand_below(afl, temp_len)] ^= 1 + rand_below(afl, 255); + out_buf[pos] ^= item; break; } - case 44 ... 46: { + case MUT_CLONE_COPY: { - if (temp_len + HAVOC_BLK_XL < MAX_FILE) { + if (likely(temp_len + HAVOC_BLK_XL < MAX_FILE)) { /* Clone bytes. */ @@ -2505,8 +2512,8 @@ havoc_stage: u32 clone_to = rand_below(afl, temp_len); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " CLONE-%s-%u-%u-%u", - "clone", clone_from, clone_to, clone_len); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " CLONE-%s_%u_%u_%u", + "COPY", clone_from, clone_to, clone_len); strcat(afl->mutation, afl->m_tmp); #endif u8 *new_buf = @@ -2529,24 +2536,35 @@ havoc_stage: afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); temp_len += clone_len; + } else if (unlikely(temp_len < 8)) { + + break; + + } else { + + goto retry_havoc_step; + } break; } - case 47: { + case MUT_CLONE_FIXED: { - if (temp_len + HAVOC_BLK_XL < MAX_FILE) { + if (likely(temp_len + HAVOC_BLK_XL < MAX_FILE)) { /* Insert a block of constant bytes (25%). */ u32 clone_len = choose_block_len(afl, HAVOC_BLK_XL); u32 clone_to = rand_below(afl, temp_len); + u32 strat = rand_below(afl, 2); + u32 clone_from = clone_to ? clone_to - 1 : 0; + item = strat ? rand_below(afl, 256) : out_buf[clone_from]; #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " CLONE-%s-%u-%u", - "insert", clone_to, clone_len); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " CLONE-%s_%u_%u_%u", + "FIXED", strat, clone_to, clone_len); strcat(afl->mutation, afl->m_tmp); #endif u8 *new_buf = @@ -2559,10 +2577,7 @@ havoc_stage: /* Inserted part */ - memset(new_buf + clone_to, - rand_below(afl, 2) ? rand_below(afl, 256) - : out_buf[rand_below(afl, temp_len)], - clone_len); + memset(new_buf + clone_to, item, clone_len); /* Tail */ memcpy(new_buf + clone_to + clone_len, out_buf + clone_to, @@ -2572,66 +2587,77 @@ havoc_stage: afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); temp_len += clone_len; + } else if (unlikely(temp_len < 8)) { + + break; + + } else { + + goto retry_havoc_step; + } break; } - case 48 ... 50: { + case MUT_OVERWRITE_COPY: { /* Overwrite bytes with a randomly selected chunk bytes. */ - if (temp_len < 2) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry - u32 copy_len = choose_block_len(afl, temp_len - 1); - u32 copy_from = rand_below(afl, temp_len - copy_len + 1); - u32 copy_to = rand_below(afl, temp_len - copy_len + 1); + u32 copy_from, copy_to, + copy_len = choose_block_len(afl, temp_len - 1); - if (likely(copy_from != copy_to)) { + do { + + copy_from = rand_below(afl, temp_len - copy_len + 1); + copy_to = rand_below(afl, temp_len - copy_len + 1); + + } while (unlikely(copy_from == copy_to)); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " OVERWRITE_COPY-%u-%u-%u", - copy_from, copy_to, copy_len); - strcat(afl->mutation, afl->m_tmp); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " OVERWRITE-COPY_%u_%u_%u", + copy_from, copy_to, copy_len); + strcat(afl->mutation, afl->m_tmp); #endif - memmove(out_buf + copy_to, out_buf + copy_from, copy_len); - - } + memmove(out_buf + copy_to, out_buf + copy_from, copy_len); break; } - case 51: { + case MUT_OVERWRITE_FIXED: { /* Overwrite bytes with fixed bytes. */ - if (temp_len < 2) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry u32 copy_len = choose_block_len(afl, temp_len - 1); u32 copy_to = rand_below(afl, temp_len - copy_len + 1); + u32 strat = rand_below(afl, 2); + u32 copy_from = copy_to ? copy_to - 1 : 0; + item = strat ? rand_below(afl, 256) : out_buf[copy_from]; #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " OVERWRITE_FIXED-%u-%u", - copy_to, copy_len); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), + " OVERWRITE-FIXED_%u_%u_%u-%u", strat, item, copy_to, + copy_len); strcat(afl->mutation, afl->m_tmp); #endif - memset(out_buf + copy_to, - rand_below(afl, 2) ? rand_below(afl, 256) - : out_buf[rand_below(afl, temp_len)], - copy_len); + memset(out_buf + copy_to, item, copy_len); break; } - case 52: { + case MUT_BYTEADD: { /* Increase byte by 1. */ #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ADDBYTE_"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " BYTEADD_"); strcat(afl->mutation, afl->m_tmp); #endif out_buf[rand_below(afl, temp_len)]++; @@ -2639,12 +2665,12 @@ havoc_stage: } - case 53: { + case MUT_BYTESUB: { /* Decrease byte by 1. */ #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " SUBBYTE_"); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " BYTESUB_"); strcat(afl->mutation, afl->m_tmp); #endif out_buf[rand_below(afl, temp_len)]--; @@ -2652,9 +2678,9 @@ havoc_stage: } - case 54: { + case MUT_FLIP8: { - /* Flip byte. */ + /* Flip byte with a XOR 0xff. This is the same as NEG. */ #ifdef INTROSPECTION snprintf(afl->m_tmp, sizeof(afl->m_tmp), " FLIP8_"); @@ -2665,9 +2691,9 @@ havoc_stage: } - case 55 ... 56: { + case MUT_SWITCH: { - if (temp_len < 4) { break; } + if (unlikely(temp_len < 4)) { break; } // no retry /* Switch bytes. */ @@ -2677,7 +2703,7 @@ havoc_stage: switch_to = rand_below(afl, temp_len); - } while (switch_from == switch_to); + } while (unlikely(switch_from == switch_to)); if (switch_from < switch_to) { @@ -2694,7 +2720,7 @@ havoc_stage: switch_len = choose_block_len(afl, MIN(switch_len, to_end)); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " SWITCH-%s-%u-%u-%u", + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " SWITCH-%s_%u_%u_%u", "switch", switch_from, switch_to, switch_len); strcat(afl->mutation, afl->m_tmp); #endif @@ -2717,12 +2743,11 @@ havoc_stage: } - // MAX_HAVOC_ENTRY = 64 - case 57 ... MAX_HAVOC_ENTRY: { + case MUT_DEL: { /* Delete bytes. */ - if (temp_len < 2) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry /* Don't delete too much. */ @@ -2730,7 +2755,7 @@ havoc_stage: u32 del_from = rand_below(afl, temp_len - del_len + 1); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " DEL-%u-%u", del_from, + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " DEL_%u_%u", del_from, del_len); strcat(afl->mutation, afl->m_tmp); #endif @@ -2743,135 +2768,401 @@ havoc_stage: } - default: + case MUT_SHUFFLE: { - r -= (MAX_HAVOC_ENTRY + 1); + /* Shuffle bytes. */ - if (afl->extras_cnt) { + if (unlikely(temp_len < 4)) { break; } // no retry - if (r < 2) { + u32 len = choose_block_len(afl, temp_len - 1); + u32 off = rand_below(afl, temp_len - len + 1); - /* Use the dictionary. */ +#ifdef INTROSPECTION + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " SHUFFLE_%u", len); + strcat(afl->mutation, afl->m_tmp); +#endif + + for (u32 i = len - 1; i > 0; i--) { + + u32 j; + do { + + j = rand_below(afl, i + 1); + + } while (unlikely(i == j)); - u32 use_extra = rand_below(afl, afl->extras_cnt); - u32 extra_len = afl->extras[use_extra].len; + unsigned char temp = out_buf[off + i]; + out_buf[off + i] = out_buf[off + j]; + out_buf[off + j] = temp; - if (extra_len > temp_len) { break; } + } + + break; + + } + + case MUT_DELONE: { + + /* Delete bytes. */ + + if (unlikely(temp_len < 2)) { break; } // no retry + + /* Don't delete too much. */ + + u32 del_len = 1; + u32 del_from = rand_below(afl, temp_len - del_len + 1); - u32 insert_at = rand_below(afl, temp_len - extra_len + 1); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " EXTRA_OVERWRITE-%u-%u", - insert_at, extra_len); - strcat(afl->mutation, afl->m_tmp); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " DELONE_%u", del_from); + strcat(afl->mutation, afl->m_tmp); #endif - memcpy(out_buf + insert_at, afl->extras[use_extra].data, - extra_len); + memmove(out_buf + del_from, out_buf + del_from + del_len, + temp_len - del_from - del_len); - break; + temp_len -= del_len; + + break; + + } - } else if (r < 4) { + case MUT_INSERTONE: { - u32 use_extra = rand_below(afl, afl->extras_cnt); - u32 extra_len = afl->extras[use_extra].len; - if (temp_len + extra_len >= MAX_FILE) { break; } + if (unlikely(temp_len < 2)) { break; } // no retry + + u32 clone_len = 1; + u32 clone_to = rand_below(afl, temp_len); + u32 strat = rand_below(afl, 2); + u32 clone_from = clone_to ? clone_to - 1 : 0; + item = strat ? rand_below(afl, 256) : out_buf[clone_from]; - u8 *ptr = afl->extras[use_extra].data; - u32 insert_at = rand_below(afl, temp_len + 1); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), " EXTRA_INSERT-%u-%u", - insert_at, extra_len); - strcat(afl->mutation, afl->m_tmp); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INSERTONE_%u_%u", strat, + clone_to); + strcat(afl->mutation, afl->m_tmp); #endif + u8 *new_buf = + afl_realloc(AFL_BUF_PARAM(out_scratch), temp_len + clone_len); + if (unlikely(!new_buf)) { PFATAL("alloc"); } - out_buf = afl_realloc(AFL_BUF_PARAM(out), temp_len + extra_len); - if (unlikely(!out_buf)) { PFATAL("alloc"); } + /* Head */ - /* Tail */ - memmove(out_buf + insert_at + extra_len, out_buf + insert_at, - temp_len - insert_at); + memcpy(new_buf, out_buf, clone_to); - /* Inserted part */ - memcpy(out_buf + insert_at, ptr, extra_len); - temp_len += extra_len; + /* Inserted part */ - break; + memset(new_buf + clone_to, item, clone_len); - } else { + /* Tail */ + memcpy(new_buf + clone_to + clone_len, out_buf + clone_to, + temp_len - clone_to); - r -= 4; + out_buf = new_buf; + afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); + temp_len += clone_len; - } + break; + + } + + case MUT_ASCIINUM: { + + if (unlikely(temp_len < 4)) { break; } // no retry + + u32 off = rand_below(afl, temp_len), off2 = off, cnt = 0; + + while (off2 + cnt < temp_len && !isdigit(out_buf[off2 + cnt])) { + + ++cnt; } - if (afl->a_extras_cnt) { + // none found, wrap + if (off2 + cnt == temp_len) { - u32 r_cmp = 2; + off2 = 0; + cnt = 0; - if (unlikely(afl->cmplog_binary && afl->queue_cur->is_ascii)) { + while (cnt < off && !isdigit(out_buf[off2 + cnt])) { - r_cmp = MUTATE_ASCII_DICT >> 1; + ++cnt; } - if (r < r_cmp) { + if (cnt == off) { - /* Use the dictionary. */ + if (temp_len < 8) { - u32 use_extra = rand_below(afl, afl->a_extras_cnt); - u32 extra_len = afl->a_extras[use_extra].len; + break; - if (extra_len > temp_len) { break; } + } else { - u32 insert_at = rand_below(afl, temp_len - extra_len + 1); -#ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), - " AUTO_EXTRA_OVERWRITE-%u-%u", insert_at, extra_len); - strcat(afl->mutation, afl->m_tmp); -#endif - memcpy(out_buf + insert_at, afl->a_extras[use_extra].data, - extra_len); + goto retry_havoc_step; + + } + + } + + } + + off = off2 + cnt; + off2 = off + 1; + + while (off2 < temp_len && isdigit(out_buf[off2])) { + + ++off2; + + } + s64 val = out_buf[off] - '0'; + for (u32 i = off + 1; i < off2; ++i) { + + val = (val * 10) + out_buf[i] - '0'; + + } + + if (off && out_buf[off - 1] == '-') { val = -val; } + + u32 strat = rand_below(afl, 8); + switch (strat) { + + case 0: + val++; + break; + case 1: + val--; + break; + case 2: + val *= 2; + break; + case 3: + val /= 2; break; + case 4: + if (likely(val && (u64)val < 0x19999999)) { - } else if (r < (r_cmp << 1)) { + val = (u64)rand_next(afl) % (u64)((u64)val * 10); + + } else { - u32 use_extra = rand_below(afl, afl->a_extras_cnt); - u32 extra_len = afl->a_extras[use_extra].len; - if (temp_len + extra_len >= MAX_FILE) { break; } + val = rand_below(afl, 256); + + } + + break; + case 5: + val += rand_below(afl, 256); + break; + case 6: + val -= rand_below(afl, 256); + break; + case 7: + val = ~(val); + break; + + } - u8 *ptr = afl->a_extras[use_extra].data; - u32 insert_at = rand_below(afl, temp_len + 1); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), - " AUTO_EXTRA_INSERT-%u-%u", insert_at, extra_len); - strcat(afl->mutation, afl->m_tmp); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " ASCIINUM_%u_%u_%u", + afl->queue_cur->is_ascii, strat, off); + strcat(afl->mutation, afl->m_tmp); #endif + // fprintf(stderr, "val: %u-%u = %ld\n", off, off2, val); + + char buf[20]; + snprintf(buf, sizeof(buf), "%" PRId64, val); - out_buf = afl_realloc(AFL_BUF_PARAM(out), temp_len + extra_len); - if (unlikely(!out_buf)) { PFATAL("alloc"); } + // fprintf(stderr, "BEFORE: %s\n", out_buf); - /* Tail */ - memmove(out_buf + insert_at + extra_len, out_buf + insert_at, - temp_len - insert_at); + u32 old_len = off2 - off; + u32 new_len = strlen(buf); - /* Inserted part */ - memcpy(out_buf + insert_at, ptr, extra_len); - temp_len += extra_len; + if (old_len == new_len) { + + memcpy(out_buf + off, buf, new_len); + + } else { + + u8 *new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), + temp_len + new_len - old_len); + if (unlikely(!new_buf)) { PFATAL("alloc"); } + + /* Head */ + + memcpy(new_buf, out_buf, off); + + /* Inserted part */ + + memcpy(new_buf + off, buf, new_len); + + /* Tail */ + memcpy(new_buf + off + new_len, out_buf + off2, temp_len - off2); + + out_buf = new_buf; + afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); + temp_len += (new_len - old_len); + + } + + // fprintf(stderr, "AFTER : %s\n", out_buf); + break; + + } + + case MUT_INSERTASCIINUM: { + + u32 len = 1 + rand_below(afl, 8); + u32 pos = rand_below(afl, temp_len); + /* Insert ascii number. */ + if (unlikely(temp_len < pos + len)) { + + if (unlikely(temp_len < 8)) { break; } else { - r -= (r_cmp << 1); + goto retry_havoc_step; } } - /* Splicing otherwise if we are still here. - Overwrite bytes with a randomly selected chunk from another - testcase or insert that chunk. */ +#ifdef INTROSPECTION + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " INSERTASCIINUM_"); + strcat(afl->mutation, afl->m_tmp); +#endif + u64 val = rand_next(afl); + char buf[20]; + snprintf(buf, sizeof(buf), "%llu", val); + memcpy(out_buf + pos, buf, len); + + break; + + } + + case MUT_EXTRA_OVERWRITE: { + + if (unlikely(!afl->extras_cnt)) { goto retry_havoc_step; } + + /* Use the dictionary. */ + + u32 use_extra = rand_below(afl, afl->extras_cnt); + u32 extra_len = afl->extras[use_extra].len; + + if (unlikely(extra_len > temp_len)) { goto retry_havoc_step; } + + u32 insert_at = rand_below(afl, temp_len - extra_len + 1); +#ifdef INTROSPECTION + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " EXTRA-OVERWRITE_%u_%u", + insert_at, extra_len); + strcat(afl->mutation, afl->m_tmp); +#endif + memcpy(out_buf + insert_at, afl->extras[use_extra].data, extra_len); + + break; + + } + + case MUT_EXTRA_INSERT: { + + if (unlikely(!afl->extras_cnt)) { goto retry_havoc_step; } + + u32 use_extra = rand_below(afl, afl->extras_cnt); + u32 extra_len = afl->extras[use_extra].len; + if (unlikely(temp_len + extra_len >= MAX_FILE)) { + + goto retry_havoc_step; + + } + + u8 *ptr = afl->extras[use_extra].data; + u32 insert_at = rand_below(afl, temp_len + 1); +#ifdef INTROSPECTION + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " EXTRA-INSERT_%u_%u", + insert_at, extra_len); + strcat(afl->mutation, afl->m_tmp); +#endif + + out_buf = afl_realloc(AFL_BUF_PARAM(out), temp_len + extra_len); + if (unlikely(!out_buf)) { PFATAL("alloc"); } + + /* Tail */ + memmove(out_buf + insert_at + extra_len, out_buf + insert_at, + temp_len - insert_at); + + /* Inserted part */ + memcpy(out_buf + insert_at, ptr, extra_len); + temp_len += extra_len; + + break; + + } + + case MUT_AUTO_EXTRA_OVERWRITE: { + + if (unlikely(!afl->a_extras_cnt)) { goto retry_havoc_step; } + + /* Use the dictionary. */ + + u32 use_extra = rand_below(afl, afl->a_extras_cnt); + u32 extra_len = afl->a_extras[use_extra].len; + + if (unlikely(extra_len > temp_len)) { goto retry_havoc_step; } + + u32 insert_at = rand_below(afl, temp_len - extra_len + 1); +#ifdef INTROSPECTION + snprintf(afl->m_tmp, sizeof(afl->m_tmp), + " AUTO-EXTRA-OVERWRITE_%u_%u", insert_at, extra_len); + strcat(afl->mutation, afl->m_tmp); +#endif + memcpy(out_buf + insert_at, afl->a_extras[use_extra].data, extra_len); + + break; + + } + + case MUT_AUTO_EXTRA_INSERT: { + + if (unlikely(!afl->a_extras_cnt)) { goto retry_havoc_step; } + + u32 use_extra = rand_below(afl, afl->a_extras_cnt); + u32 extra_len = afl->a_extras[use_extra].len; + if (unlikely(temp_len + extra_len >= MAX_FILE)) { + + goto retry_havoc_step; + + } + + u8 *ptr = afl->a_extras[use_extra].data; + u32 insert_at = rand_below(afl, temp_len + 1); +#ifdef INTROSPECTION + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " AUTO-EXTRA-INSERT_%u_%u", + insert_at, extra_len); + strcat(afl->mutation, afl->m_tmp); +#endif + + out_buf = afl_realloc(AFL_BUF_PARAM(out), temp_len + extra_len); + if (unlikely(!out_buf)) { PFATAL("alloc"); } + + /* Tail */ + memmove(out_buf + insert_at + extra_len, out_buf + insert_at, + temp_len - insert_at); + + /* Inserted part */ + memcpy(out_buf + insert_at, ptr, extra_len); + temp_len += extra_len; + + break; + + } + + case MUT_SPLICE_OVERWRITE: { + + if (unlikely(afl->ready_for_splicing_count <= 1)) { + + goto retry_havoc_step; + + } /* Pick a random queue entry and seek to it. */ @@ -2880,79 +3171,110 @@ havoc_stage: tid = rand_below(afl, afl->queued_items); - } while (tid == afl->current_entry || afl->queue_buf[tid]->len < 4); + } while (unlikely(tid == afl->current_entry || + + afl->queue_buf[tid]->len < 4)); /* Get the testcase for splicing. */ struct queue_entry *target = afl->queue_buf[tid]; u32 new_len = target->len; u8 *new_buf = queue_testcase_get(afl, target); - if ((temp_len >= 2 && r % 2) || temp_len + HAVOC_BLK_XL >= MAX_FILE) { - - /* overwrite mode */ + /* overwrite mode */ - u32 copy_from, copy_to, copy_len; + u32 copy_from, copy_to, copy_len; - copy_len = choose_block_len(afl, new_len - 1); - if (copy_len > temp_len) copy_len = temp_len; + copy_len = choose_block_len(afl, new_len - 1); + if (copy_len > temp_len) copy_len = temp_len; - copy_from = rand_below(afl, new_len - copy_len + 1); - copy_to = rand_below(afl, temp_len - copy_len + 1); + copy_from = rand_below(afl, new_len - copy_len + 1); + copy_to = rand_below(afl, temp_len - copy_len + 1); #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), - " SPLICE_OVERWRITE-%u-%u-%u-%s", copy_from, copy_to, - copy_len, target->fname); - strcat(afl->mutation, afl->m_tmp); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), + " SPLICE-OVERWRITE_%u_%u_%u_%s", copy_from, copy_to, + copy_len, target->fname); + strcat(afl->mutation, afl->m_tmp); #endif - memmove(out_buf + copy_to, new_buf + copy_from, copy_len); + memmove(out_buf + copy_to, new_buf + copy_from, copy_len); - } else { + break; + + } + + case MUT_SPLICE_INSERT: { + + if (unlikely(afl->ready_for_splicing_count <= 1)) { + + goto retry_havoc_step; + + } + + if (unlikely(temp_len + HAVOC_BLK_XL >= MAX_FILE)) { + + goto retry_havoc_step; + + } + + /* Pick a random queue entry and seek to it. */ + + u32 tid; + do { + + tid = rand_below(afl, afl->queued_items); + + } while (unlikely(tid == afl->current_entry || + + afl->queue_buf[tid]->len < 4)); - /* insert mode */ + /* Get the testcase for splicing. */ + struct queue_entry *target = afl->queue_buf[tid]; + u32 new_len = target->len; + u8 *new_buf = queue_testcase_get(afl, target); - u32 clone_from, clone_to, clone_len; + /* insert mode */ - clone_len = choose_block_len(afl, new_len); - clone_from = rand_below(afl, new_len - clone_len + 1); - clone_to = rand_below(afl, temp_len + 1); + u32 clone_from, clone_to, clone_len; - u8 *temp_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), - temp_len + clone_len + 1); - if (unlikely(!temp_buf)) { PFATAL("alloc"); } + clone_len = choose_block_len(afl, new_len); + clone_from = rand_below(afl, new_len - clone_len + 1); + clone_to = rand_below(afl, temp_len + 1); + + u8 *temp_buf = + afl_realloc(AFL_BUF_PARAM(out_scratch), temp_len + clone_len + 1); + if (unlikely(!temp_buf)) { PFATAL("alloc"); } #ifdef INTROSPECTION - snprintf(afl->m_tmp, sizeof(afl->m_tmp), - " SPLICE_INSERT-%u-%u-%u-%s", clone_from, clone_to, - clone_len, target->fname); - strcat(afl->mutation, afl->m_tmp); + snprintf(afl->m_tmp, sizeof(afl->m_tmp), " SPLICE-INSERT_%u_%u_%u_%s", + clone_from, clone_to, clone_len, target->fname); + strcat(afl->mutation, afl->m_tmp); #endif - /* Head */ + /* Head */ - memcpy(temp_buf, out_buf, clone_to); + memcpy(temp_buf, out_buf, clone_to); - /* Inserted part */ - - memcpy(temp_buf + clone_to, new_buf + clone_from, clone_len); + /* Inserted part */ - /* Tail */ - memcpy(temp_buf + clone_to + clone_len, out_buf + clone_to, - temp_len - clone_to); + memcpy(temp_buf + clone_to, new_buf + clone_from, clone_len); - out_buf = temp_buf; - afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); - temp_len += clone_len; + /* Tail */ + memcpy(temp_buf + clone_to + clone_len, out_buf + clone_to, + temp_len - clone_to); - } + out_buf = temp_buf; + afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); + temp_len += clone_len; break; - // end of default + } } } + } + if (common_fuzz_stuff(afl, out_buf, temp_len)) { goto abandon_entry; } /* out_buf might have been mangled a bit, so let's restore it to its @@ -3038,7 +3360,9 @@ retry_splicing: tid = rand_below(afl, afl->queued_items); - } while (tid == afl->current_entry || afl->queue_buf[tid]->len < 4); + } while ( + + unlikely(tid == afl->current_entry || afl->queue_buf[tid]->len < 4)); /* Get the testcase */ afl->splicing_with = tid; @@ -3078,6 +3402,25 @@ retry_splicing: ret_val = 0; +#ifdef INTROSPECTION + + afl->havoc_prof->queued_det_stage = + before_havoc_findings - before_det_findings; + afl->havoc_prof->queued_havoc_stage = + afl->queued_items - before_havoc_findings; + afl->havoc_prof->total_queued_det += afl->havoc_prof->queued_det_stage; + afl->havoc_prof->edge_det_stage = before_havoc_edges - before_det_edges; + afl->havoc_prof->edge_havoc_stage = + count_non_255_bytes(afl, afl->virgin_bits) - before_havoc_edges; + afl->havoc_prof->total_det_edge += afl->havoc_prof->edge_det_stage; + afl->havoc_prof->det_stage_time = before_havoc_time - before_det_time; + afl->havoc_prof->havoc_stage_time = get_cur_time() - before_havoc_time; + afl->havoc_prof->total_det_time += afl->havoc_prof->det_stage_time; + + plot_profile_data(afl, afl->queue_cur); + +#endif + /* we are through with this queue entry - for this iteration */ abandon_entry: @@ -3092,7 +3435,12 @@ abandon_entry: --afl->pending_not_fuzzed; afl->queue_cur->was_fuzzed = 1; afl->reinit_table = 1; - if (afl->queue_cur->favored) { --afl->pending_favored; } + if (afl->queue_cur->favored) { + + --afl->pending_favored; + afl->smallest_favored = -1; + + } } @@ -3348,13 +3696,13 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { * SIMPLE BITFLIP (+dictionary construction) * *********************************************/ -#define FLIP_BIT(_ar, _b) \ - do { \ - \ - u8 *_arf = (u8 *)(_ar); \ - u32 _bf = (_b); \ - _arf[(_bf) >> 3] ^= (128 >> ((_bf)&7)); \ - \ +#define FLIP_BIT(_ar, _b) \ + do { \ + \ + u8 *_arf = (u8 *)(_ar); \ + u32 _bf = (_b); \ + _arf[(_bf) >> 3] ^= (128 >> ((_bf) & 7)); \ + \ } while (0) /* Single walking bit. */ @@ -5555,7 +5903,13 @@ pacemaker_fuzzing: --afl->pending_not_fuzzed; afl->queue_cur->was_fuzzed = 1; - if (afl->queue_cur->favored) { --afl->pending_favored; } + afl->reinit_table = 1 + if (afl->queue_cur->favored) { + + --afl->pending_favored; + afl->smallest_favored = -1; + + } } diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 7dad0770..873b25e2 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -249,6 +249,8 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { PyObject_GetAttrString(py_module, "queue_get"); py_functions[PY_FUNC_FUZZ_SEND] = PyObject_GetAttrString(py_module, "fuzz_send"); + py_functions[PY_FUNC_POST_RUN] = + PyObject_GetAttrString(py_module, "post_run"); py_functions[PY_FUNC_SPLICE_OPTOUT] = PyObject_GetAttrString(py_module, "splice_optout"); if (py_functions[PY_FUNC_SPLICE_OPTOUT]) { afl->custom_splice_optout = 1; } @@ -468,6 +470,12 @@ struct custom_mutator *load_custom_mutator_py(afl_state_t *afl, } + if (py_functions[PY_FUNC_POST_RUN]) { + + mutator->afl_custom_post_run = post_run_py; + + } + if (py_functions[PY_FUNC_SPLICE_OPTOUT]) { mutator->afl_custom_splice_optout = splice_optout_py; @@ -925,6 +933,28 @@ void fuzz_send_py(void *py_mutator, const u8 *buf, size_t buf_size) { } +void post_run_py(void *py_mutator) { + + PyObject *py_args, *py_value; + + py_args = PyTuple_New(0); + py_value = PyObject_CallObject( + ((py_mutator_t *)py_mutator)->py_functions[PY_FUNC_POST_RUN], py_args); + Py_DECREF(py_args); + + if (py_value != NULL) { + + Py_DECREF(py_value); + + } else { + + PyErr_Print(); + FATAL("Call failed"); + + } + +} + u8 queue_new_entry_py(void *py_mutator, const u8 *filename_new_queue, const u8 *filename_orig_queue) { diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index b10bf749..2318df60 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: @@ -80,6 +80,7 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q, if (unlikely(weight < 0.1)) { weight = 0.1; } if (unlikely(q->favored)) { weight *= 5; } if (unlikely(!q->was_fuzzed)) { weight *= 2; } + if (unlikely(q->fs_redundant)) { weight *= 0.8; } return weight; @@ -369,9 +370,9 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) { s32 fd; - fd = open(fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); - if (fd < 0) { PFATAL("Unable to create '%s'", fn); } - close(fd); + if (unlikely(afl->afl_env.afl_disable_redundant)) { q->disabled = 1; } + fd = permissive_create(afl, fn); + if (fd >= 0) { close(fd); } } else { @@ -612,7 +613,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { } - if (likely(q->len > 4)) afl->ready_for_splicing_count++; + if (likely(q->len > 4)) { ++afl->ready_for_splicing_count; } ++afl->queued_items; ++afl->active_items; @@ -663,6 +664,8 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { } + q->skipdet_e = (struct skipdet_entry *)ck_alloc(sizeof(struct skipdet_entry)); + } /* Destroy the entire queue. */ @@ -678,6 +681,15 @@ void destroy_queue(afl_state_t *afl) { q = afl->queue_buf[i]; ck_free(q->fname); ck_free(q->trace_mini); + if (q->skipdet_e) { + + if (q->skipdet_e->done_inf_map) ck_free(q->skipdet_e->done_inf_map); + if (q->skipdet_e->skip_eff_map) ck_free(q->skipdet_e->skip_eff_map); + + ck_free(q->skipdet_e); + + } + ck_free(q); } @@ -701,13 +713,20 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { u64 fav_factor; u64 fuzz_p2; - if (unlikely(afl->schedule >= FAST && afl->schedule < RARE)) + if (likely(afl->schedule >= FAST && afl->schedule < RARE)) { + fuzz_p2 = 0; // Skip the fuzz_p2 comparison - else if (unlikely(afl->schedule == RARE)) + + } else if (unlikely(afl->schedule == RARE)) { + fuzz_p2 = next_pow2(afl->n_fuzz[q->n_fuzz_entry]); - else + + } else { + fuzz_p2 = q->fuzz_level; + } + if (unlikely(afl->schedule >= RARE) || unlikely(afl->fixed_seed)) { fav_factor = q->len << 2; @@ -729,47 +748,36 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { /* Faster-executing or smaller test cases are favored. */ u64 top_rated_fav_factor; u64 top_rated_fuzz_p2; - if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) - top_rated_fuzz_p2 = - next_pow2(afl->n_fuzz[afl->top_rated[i]->n_fuzz_entry]); - else - top_rated_fuzz_p2 = afl->top_rated[i]->fuzz_level; - - if (unlikely(afl->schedule >= RARE) || unlikely(afl->fixed_seed)) { - - top_rated_fav_factor = afl->top_rated[i]->len << 2; - } else { - - top_rated_fav_factor = - afl->top_rated[i]->exec_us * afl->top_rated[i]->len; + if (likely(afl->schedule >= FAST && afl->schedule < RARE)) { - } + top_rated_fuzz_p2 = 0; // Skip the fuzz_p2 comparison - if (fuzz_p2 > top_rated_fuzz_p2) { + } else if (unlikely(afl->schedule == RARE)) { - continue; + top_rated_fuzz_p2 = + next_pow2(afl->n_fuzz[afl->top_rated[i]->n_fuzz_entry]); - } else if (fuzz_p2 == top_rated_fuzz_p2) { + } else { - if (fav_factor > top_rated_fav_factor) { continue; } + top_rated_fuzz_p2 = afl->top_rated[i]->fuzz_level; } if (unlikely(afl->schedule >= RARE) || unlikely(afl->fixed_seed)) { - if (fav_factor > afl->top_rated[i]->len << 2) { continue; } + top_rated_fav_factor = afl->top_rated[i]->len << 2; } else { - if (fav_factor > - afl->top_rated[i]->exec_us * afl->top_rated[i]->len) { + top_rated_fav_factor = + afl->top_rated[i]->exec_us * afl->top_rated[i]->len; - continue; + } - } + if (likely(fuzz_p2 > top_rated_fuzz_p2)) { continue; } - } + if (likely(fav_factor > top_rated_fav_factor)) { continue; } /* Looks like we're going to win. Decrease ref count for the previous winner, discard its afl->fsrv.trace_bits[] if necessary. */ @@ -834,6 +842,8 @@ void cull_queue(afl_state_t *afl) { /* Let's see if anything in the bitmap isn't captured in temp_v. If yes, and if it has a afl->top_rated[] contender, let's use it. */ + afl->smallest_favored = -1; + for (i = 0; i < afl->fsrv.map_size; ++i) { if (afl->top_rated[i] && (temp_v[i >> 3] & (1 << (i & 7)))) { @@ -857,7 +867,16 @@ void cull_queue(afl_state_t *afl) { afl->top_rated[i]->favored = 1; ++afl->queued_favored; - if (!afl->top_rated[i]->was_fuzzed) { ++afl->pending_favored; } + if (!afl->top_rated[i]->was_fuzzed) { + + ++afl->pending_favored; + if (unlikely(afl->smallest_favored < 0)) { + + afl->smallest_favored = (s64)afl->top_rated[i]->id; + + } + + } } @@ -875,6 +894,8 @@ void cull_queue(afl_state_t *afl) { } + afl->reinit_table = 1; + } /* Calculate case desirability score to adjust the length of havoc fuzzing. diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 6e4a655b..9316da71 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -7,7 +7,7 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. @@ -28,8 +28,9 @@ #include "afl-fuzz.h" #include "cmplog.h" -//#define _DEBUG -//#define CMPLOG_INTROSPECTION +// #define _DEBUG +// #define USE_HASHMAP +// #define CMPLOG_INTROSPECTION // CMP attribute enum enum { @@ -40,7 +41,7 @@ enum { IS_FP = 8, // is a floating point, not an integer /* --- below are internal settings, not from target cmplog */ IS_FP_MOD = 16, // arithemtic changed floating point - IS_INT_MOD = 32, // arithmetic changed interger + IS_INT_MOD = 32, // arithmetic changed integer IS_TRANSFORM = 64 // transformed integer }; @@ -87,6 +88,13 @@ static u32 hshape; static u64 screen_update; static u64 last_update; +#ifdef USE_HASHMAP +// hashmap functions +void hashmap_reset(); +bool hashmap_search_and_add(uint8_t type, uint64_t key); +bool hashmap_search_and_add_ptr(uint8_t type, u8 *key); +#endif + static struct range *add_range(struct range *ranges, u32 start, u32 end) { struct range *r = ck_alloc_nozero(sizeof(struct range)); @@ -129,7 +137,6 @@ static struct range *pop_biggest_range(struct range **ranges) { } #ifdef _DEBUG -// static int logging = 0; static void dump(char *txt, u8 *buf, u32 len) { u32 i; @@ -140,6 +147,7 @@ static void dump(char *txt, u8 *buf, u32 len) { } +/* static void dump_file(char *path, char *name, u32 counter, u8 *buf, u32 len) { char fn[4096]; @@ -155,6 +163,8 @@ static void dump_file(char *path, char *name, u32 counter, u8 *buf, u32 len) { } +*/ + #endif static u8 get_exec_checksum(afl_state_t *afl, u8 *buf, u32 len, u64 *cksum) { @@ -379,7 +389,7 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, } - if (++afl->stage_cur % screen_update == 0) { show_stats(afl); }; + if (unlikely(++afl->stage_cur % screen_update == 0)) { show_stats(afl); }; } @@ -571,7 +581,6 @@ static u8 its_fuzz(afl_state_t *afl, u8 *buf, u32 len, u8 *status) { } -//#ifdef CMPLOG_SOLVE_TRANSFORM static int strntoll(const char *str, size_t sz, char **end, int base, long long *out) { @@ -656,7 +665,6 @@ static int is_hex(const char *str) { } -#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64 // tests 4 bytes at location static int is_base64(const char *str) { @@ -732,12 +740,14 @@ static u32 from_base64(u8 *src, u8 *dst, u32 dst_len) { } -static void to_base64(u8 *src, u8 *dst, u32 dst_len) { +static u32 to_base64(u8 *src, u8 *dst, u32 dst_len) { u32 i, j, v; - u32 len = (dst_len >> 2) * 3; + // u32 len = (dst_len >> 2) * 3; + u32 len = (dst_len / 3) * 4; + if (dst_len % 3) len += 4; - for (i = 0, j = 0; i < len; i += 3, j += 4) { + for (i = 0, j = 0; j < len; i += 3, j += 4) { v = src[i]; v = i + 1 < len ? v << 8 | src[i + 1] : v << 8; @@ -745,7 +755,8 @@ static void to_base64(u8 *src, u8 *dst, u32 dst_len) { dst[j] = base64_encode_table[(v >> 18) & 0x3F]; dst[j + 1] = base64_encode_table[(v >> 12) & 0x3F]; - if (i + 1 < len) { + + if (i + 1 < dst_len) { dst[j + 2] = base64_encode_table[(v >> 6) & 0x3F]; @@ -755,7 +766,7 @@ static void to_base64(u8 *src, u8 *dst, u32 dst_len) { } - if (i + 2 < len) { + if (i + 2 < dst_len) { dst[j + 3] = base64_encode_table[v & 0x3F]; @@ -767,12 +778,18 @@ static void to_base64(u8 *src, u8 *dst, u32 dst_len) { } + dst[len] = 0; + return len; + } +#ifdef WORD_SIZE_64 +static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, + u128 pattern, u128 repl, u128 o_pattern, + u128 changed_val, u8 attr, u32 idx, + u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf, + u32 len, u8 do_reverse, u8 lvl, u8 *status); #endif - -//#endif - static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u64 pattern, u64 repl, u64 o_pattern, u64 changed_val, u8 attr, u32 idx, u32 taint_len, @@ -786,53 +803,89 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u64 *o_buf_64 = (u64 *)&orig_buf[idx]; u32 *o_buf_32 = (u32 *)&orig_buf[idx]; u16 *o_buf_16 = (u16 *)&orig_buf[idx]; - u8 *o_buf_8 = &orig_buf[idx]; + // u8 *o_buf_8 = &orig_buf[idx]; u32 its_len = MIN(len - idx, taint_len); - if (afl->fsrv.total_execs - last_update > screen_update) { + if (unlikely(afl->fsrv.total_execs - last_update > screen_update)) { show_stats(afl); last_update = afl->fsrv.total_execs; } - // fprintf(stderr, - // "Encode: %llx->%llx into %llx(<-%llx) at idx=%u " - // "taint_len=%u shape=%u attr=%u\n", - // o_pattern, pattern, repl, changed_val, idx, taint_len, - // hshape, attr); + /* + fprintf(stderr, + "Encode: %llx->%llx into %llx(<-%llx) at idx=%u " + "taint_len=%u shape=%u attr=%u\n", + o_pattern, pattern, repl, changed_val, idx, taint_len, + hshape, attr); + */ + + u8 bytes; + + switch (hshape) { - //#ifdef CMPLOG_SOLVE_TRANSFORM - // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3 + case 0: + case 1: + bytes = 1; + break; + case 2: + bytes = 2; + break; + case 3: + case 4: + bytes = 4; + break; + default: + bytes = 8; + + } + + // necessary for preventing heap access overflow + bytes = MIN(bytes, len - idx); + if (unlikely(bytes <= 1)) { return 0; } + + // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3 if (afl->cmplog_enable_transform && (lvl & LVL3)) { u8 *endptr; u8 use_num = 0, use_unum = 0; - unsigned long long unum; - long long num; + unsigned long long unum = 0; + long long num = 0; + + // if (afl->queue_cur->is_ascii) { - if (afl->queue_cur->is_ascii) { + // we first check if our input are ascii numbers that are transformed to + // an integer and used for comparison: - endptr = buf_8; - if (strntoll(buf_8, len - idx, (char **)&endptr, 0, &num)) { + endptr = buf_8; + if (strntoll(buf_8, len - idx, (char **)&endptr, 0, &num)) { - if (!strntoull(buf_8, len - idx, (char **)&endptr, 0, &unum)) - use_unum = 1; + if (!strntoull(buf_8, len - idx, (char **)&endptr, 0, &unum)) { - } else + use_unum = 1; - use_num = 1; + } + + } else { + + use_num = 1; } + //} + #ifdef _DEBUG if (idx == 0) - fprintf(stderr, "ASCII is=%u use_num=%u use_unum=%u idx=%u %llx==%llx\n", - afl->queue_cur->is_ascii, use_num, use_unum, idx, num, pattern); + fprintf(stderr, + "ASCII is=%u use_num=%u>%lld use_unum=%u>%llu idx=%u " + "pattern=0x%llx\n", + afl->queue_cur->is_ascii, use_num, num, use_unum, unum, idx, + pattern); #endif - // num is likely not pattern as atoi("AAA") will be zero... + // atoi("AAA") == 0 so !num means we have to investigate if (use_num && ((u64)num == pattern || !num)) { u8 tmp_buf[32]; @@ -881,29 +934,6 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, if (pattern != o_pattern && repl == changed_val && attr <= IS_EQUAL) { u64 b_val, o_b_val, mask; - u8 bytes; - - switch (hshape) { - - case 0: - case 1: - bytes = 1; - break; - case 2: - bytes = 2; - break; - case 3: - case 4: - bytes = 4; - break; - default: - bytes = 8; - - } - - // necessary for preventing heap access overflow - bytes = MIN(bytes, len - idx); - switch (bytes) { case 0: // cannot happen @@ -961,10 +991,12 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, // test for arithmetic, eg. "if ((user_val - 0x1111) == 0x1234) ..." s64 diff = pattern - b_val; s64 o_diff = o_pattern - o_b_val; - /* fprintf(stderr, "DIFF1 idx=%03u shape=%02u %llx-%llx=%lx\n", idx, - hshape, o_pattern, o_b_val, o_diff); - fprintf(stderr, "DIFF1 %016llx %llx-%llx=%lx\n", repl, pattern, - b_val, diff); */ + /* + fprintf(stderr, "DIFF1 idx=%03u shape=%02u %llx-%llx=%lx\n", idx, + hshape, o_pattern, o_b_val, o_diff); + fprintf(stderr, "DIFF1 %016llx %llx-%llx=%lx\n", repl, pattern, + b_val, diff); + */ if (diff == o_diff && diff) { // this could be an arithmetic transformation @@ -1120,7 +1152,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - //#endif + // #endif // we only allow this for ascii2integer (above) so leave if this is the case if (unlikely(pattern == o_pattern)) { return 0; } @@ -1243,6 +1275,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } + /* if (*status != 1) { // u8 // if (its_len >= 1) @@ -1267,15 +1300,145 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } + */ + } - // here we add and subract 1 from the value, but only if it is not an + // If 'S' is set for cmplog mode then we try a scale encoding of the value. + // Currently we can only handle bytes up to 1 << 55 on 32 bit and 1 << 119 + // on 64 bit systems. + // Caveat: This implementation here works only on little endian systems. + + if (attr < IS_FP && (afl->cmplog_enable_scale || lvl >= LVL3) && + repl == changed_val) { + + u8 do_call = 1; + u64 new_val = repl << 2; + u32 ilen = 0; + + if (changed_val <= 255) { + + ilen = 1; + + } else if (new_val <= 65535) { + + new_val += 1; // two byte mode + ilen = 2; + + } else if (new_val <= 4294967295) { + + new_val += 2; // four byte mode + ilen = 4; + + } else { + +#ifndef WORD_SIZE_64 + if (repl <= 0x00ffffffffffffff) { + + new_val = repl << 8; + u8 scale_len = 0; + u64 tmp_val = repl; + while (tmp_val) { + + tmp_val >>= 8; + ++scale_len; + + } // scale_len will be >= 4; + + if (scale_len >= 4) { + + scale_len -= 4; + + } else { + + scale_len = 0; + + }; + + new_val += (scale_len << 2) + 3; + ilen = scale_len + 5; + + } else { + + do_call = 0; + + } + +#else + { + + u128 new_vall = ((u128)repl) << 8; + u8 scale_len = 0; + u128 tmp_val = (u128)repl; + + while (tmp_val) { + + tmp_val >>= 8; + ++scale_len; + + } // scale_len will be >= 4; + + if (scale_len >= 4) { + + scale_len -= 4; + + } else { + + scale_len = 0; + + }; + + new_vall += (scale_len << 2) + 3; + ilen = scale_len + 5; + + if (ilen <= its_len && ilen > 1) { + + u8 tmpbuf[32]; + memcpy(tmpbuf, buf + idx, ilen); + memcpy(buf + idx, (char *)&new_vall, ilen); + + if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } + #ifdef CMPLOG_COMBINE + if (*status == 1) { memcpy(cbuf + idx, (char *)&new_vall, ilen); } + #endif + memcpy(buf + idx, tmpbuf, ilen); + + }; + + do_call = 0; + + } + +#endif + + } + + if (do_call) { + + if (ilen <= its_len && ilen > 1) { + + u8 tmpbuf[32]; + memcpy(tmpbuf, buf + idx, ilen); + memcpy(buf + idx, (char *)&new_val, ilen); + + if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } +#ifdef CMPLOG_COMBINE + if (*status == 1) { memcpy(cbuf + idx, (char *)&new_val, ilen); } +#endif + memcpy(buf + idx, tmpbuf, ilen); + + }; + + } + + } + + // here we add and subtract 1 from the value, but only if it is not an // == or != comparison // Bits: 1 = Equal, 2 = Greater, 4 = Lesser, 8 = Float // 16 = modified float, 32 = modified integer (modified = wont match // in original buffer) - //#ifdef CMPLOG_SOLVE_ARITHMETIC if (!afl->cmplog_enable_arith || lvl < LVL3 || attr == IS_TRANSFORM) { return 0; @@ -1440,8 +1603,8 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - //#endif /* - // CMPLOG_SOLVE_ARITHMETIC + // #endif /* + // CMPLOG_SOLVE_ARITHMETIC return 0; @@ -1455,7 +1618,7 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf, u32 len, u8 do_reverse, u8 lvl, u8 *status) { - if (afl->fsrv.total_execs - last_update > screen_update) { + if (unlikely(afl->fsrv.total_execs - last_update > screen_update)) { show_stats(afl); last_update = afl->fsrv.total_execs; @@ -1536,6 +1699,77 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, } + // Scale encoding only works on little endian systems + + if (attr < IS_FP && attr < 32 && + (afl->cmplog_enable_scale || lvl >= LVL3)) { + + u128 new_val = repl << 2; + u128 max_scale = (u128)1 << 120; + u32 ilen = 0; + u8 do_call = 1; + + if (new_val <= 255) { + + ilen = 1; + + } else if (new_val <= 65535) { + + new_val += 1; // two byte mode + ilen = 2; + + } else if (new_val <= 4294967295) { + + new_val += 2; // four byte mode + ilen = 4; + + } else if (repl < max_scale) { + + new_val = (u128)repl << 8; + u8 scale_len = 0; + u128 tmp_val = (u128)repl; + while (tmp_val) { + + tmp_val >>= 8; + ++scale_len; + + } // scale_len will be >= 4; + + if (scale_len >= 4) { + + scale_len -= 4; + + } else { + + scale_len = 0; + + }; + + new_val += (scale_len << 2) + 3; + ilen = scale_len + 5; + + } else { + + do_call = 0; + + } + + if (do_call && ilen <= its_len) { + + u8 tmpbuf[32]; + memcpy(tmpbuf, buf + idx, ilen); + memcpy(buf + idx, (char *)&new_val, ilen); + + if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } + #ifdef CMPLOG_COMBINE + if (*status == 1) { memcpy(cbuf + idx, (char *)&new_val, ilen); } + #endif + memcpy(buf + idx, tmpbuf, ilen); + + }; + + } + } return 0; @@ -1606,7 +1840,7 @@ static void try_to_add_to_dictN(afl_state_t *afl, u128 v, u8 size) { for (k = 0; k < size; ++k) { #else - u32 off = 16 - size; + u32 off = 16 - size; for (k = 16 - size; k < 16; ++k) { #endif @@ -1659,6 +1893,8 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, hshape = SHAPE_BYTES(h->shape); + if (hshape < 2) { return 0; } + if (h->hits > CMP_MAP_H) { loggeds = CMP_MAP_H; @@ -1721,6 +1957,19 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } +#ifdef USE_HASHMAP + // TODO: add attribute? not sure + if (hshape <= 8 && hashmap_search_and_add(hshape - 1, o->v0) && + hashmap_search_and_add(hshape - 1, orig_o->v0) && + hashmap_search_and_add(hshape - 1, o->v1) && + hashmap_search_and_add(hshape - 1, orig_o->v1)) { + + continue; + + } + +#endif + #ifdef _DEBUG fprintf(stderr, "Handling: %llx->%llx vs %llx->%llx attr=%u shape=%u\n", orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, hshape); @@ -1948,11 +2197,11 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, #ifndef CMPLOG_COMBINE (void)(cbuf); #endif - //#ifndef CMPLOG_SOLVE_TRANSFORM - // (void)(changed_val); - //#endif + // #ifndef CMPLOG_SOLVE_TRANSFORM + // (void)(changed_val); + // #endif - if (afl->fsrv.total_execs - last_update > screen_update) { + if (unlikely(afl->fsrv.total_execs - last_update > screen_update)) { show_stats(afl); last_update = afl->fsrv.total_execs; @@ -1988,29 +2237,35 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, if (l0 >= 0x80 || ol0 >= 0x80) { - l0 -= 0x80; - l1 -= 0x80; - ol0 -= 0x80; - ol1 -= 0x80; + if (l0 >= 0x80) { l0 -= 0x80; } + if (l1 >= 0x80) { l1 -= 0x80; } + if (ol0 >= 0x80) { ol0 -= 0x80; } + if (ol1 >= 0x80) { ol1 -= 0x80; } } - if (l0 == 0 || l1 == 0 || ol0 == 0 || ol1 == 0 || l0 > 31 || l1 > 31 || - ol0 > 31 || ol1 > 31) { + if (l0 == 0 || l1 == 0 || ol0 == 0 || ol1 == 0 || l0 > 32 || l1 > 32 || + ol0 > 32 || ol1 > 32) { l0 = ol0 = hshape; } u8 lmax = MAX(l0, ol0); - u8 save[40]; + u8 save[80]; u32 saved_idx = idx, pre, from = 0, to = 0, i, j; u32 its_len = MIN(MIN(lmax, hshape), len - idx); its_len = MIN(its_len, taint_len); u32 saved_its_len = its_len; + // fprintf(stderr, "its_len=%u repl=%s\n", its_len, repl); + + if (its_len <= 1) { return 0; } + if (lvl & LVL3) { + if (memcmp(changed_val, repl, its_len) != 0) { return 0; } + u32 max_to = MIN(4U, idx); if (!(lvl & LVL1) && max_to) { from = 1; } to = max_to; @@ -2021,27 +2276,32 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, (void)(j); #ifdef _DEBUG - fprintf(stderr, "RTN T idx=%u lvl=%02x is_txt=%u shape=%u/%u ", idx, lvl, - o->v0_len >= 0x80 ? 1 : 0, hshape, l0); - for (j = 0; j < 8; j++) - fprintf(stderr, "%02x", orig_buf[idx + j]); - fprintf(stderr, " -> "); - for (j = 0; j < 8; j++) - fprintf(stderr, "%02x", o_pattern[j]); - fprintf(stderr, " <= "); - for (j = 0; j < 8; j++) - fprintf(stderr, "%02x", repl[j]); - fprintf(stderr, "\n"); - fprintf(stderr, " "); - for (j = 0; j < 8; j++) - fprintf(stderr, "%02x", buf[idx + j]); - fprintf(stderr, " -> "); - for (j = 0; j < 8; j++) - fprintf(stderr, "%02x", pattern[j]); - fprintf(stderr, " <= "); - for (j = 0; j < 8; j++) - fprintf(stderr, "%02x", changed_val[j]); - fprintf(stderr, "\n"); + if (idx == 0) { + + fprintf(stderr, "RTN T idx=%u lvl=%02x is_txt=%u shape=%u/%u ", idx, lvl, + o->v0_len >= 0x80 ? 1 : 0, hshape, l0); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", orig_buf[idx + j]); + fprintf(stderr, " -> "); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", o_pattern[j]); + fprintf(stderr, " <= "); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", repl[j]); + fprintf(stderr, "\n"); + fprintf(stderr, " "); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", buf[idx + j]); + fprintf(stderr, " -> "); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", pattern[j]); + fprintf(stderr, " <= "); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", changed_val[j]); + fprintf(stderr, "\n"); + + } + #endif // Try to match the replace value up to 4 bytes before the current idx. @@ -2050,6 +2310,9 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, // if (memcmp(user_val, "TEST-VALUE") == 0) ... // We only do this in lvl 3, otherwise we only do direct matching + // fprintf(stderr, "XXXX FROMB64 saved_idx=%u its_len=%u from=%u to=%u FROMHEX + // repl=%s\n", saved_idx, saved_its_len, from, to, repl); + for (pre = from; pre <= to; pre++) { if (*status != 1 && (!pre || !memcmp(buf + saved_idx - pre, repl, pre))) { @@ -2059,7 +2322,7 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, for (i = 0; i < its_len; ++i) { - if ((pattern[i] != buf[idx + i] && o_pattern[i] != orig_buf[idx + i]) || + if ((pattern[i] != buf[idx + i] || o_pattern[i] != orig_buf[idx + i]) || *status == 1) { break; @@ -2089,12 +2352,10 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, if (afl->cmplog_enable_transform && (lvl & LVL3)) { u32 toupper = 0, tolower = 0, xor = 0, arith = 0, tohex = 0, fromhex = 0; -#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64 u32 tob64 = 0, fromb64 = 0; -#endif u32 from_0 = 0, from_x = 0, from_X = 0, from_slash = 0, from_up = 0; u32 to_0 = 0, to_x = 0, to_slash = 0, to_up = 0; - u8 xor_val[32], arith_val[32], tmp[48]; + u8 xor_val[64], arith_val[64], tmp[64]; idx = saved_idx; its_len = saved_its_len; @@ -2144,7 +2405,8 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - if (i < 16 && is_hex(repl + (i << 1))) { + if (afl->cmplog_enable_xtreme_transform && i < 16 && + is_hex(repl + (i << 1))) { ++tohex; @@ -2163,9 +2425,9 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - if ((i % 2)) { + if (afl->cmplog_enable_xtreme_transform && (i % 2) == 1) { - if (len > idx + i + 1 && is_hex(orig_buf + idx + i)) { + if (len > idx + i + 1 && is_hex(orig_buf + idx + i - 1)) { fromhex += 2; @@ -2187,20 +2449,23 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } -#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64 - if (i % 3 == 2 && i < 24) { + if (afl->cmplog_enable_xtreme_transform) { - if (is_base64(repl + ((i / 3) << 2))) tob64 += 3; + if (i % 3 == 2 && i < 24) { - } + if (is_base64(repl + ((i / 3) << 2))) tob64 += 3; - if (i % 4 == 3 && i < 24) { + } - if (is_base64(orig_buf + idx + i - 3)) fromb64 += 4; + // fprintf(stderr, "X FROMB64 idx=%u i=%u repl=%s\n", saved_idx, i, + // repl); + if (i % 4 == 3 && i < 24) { - } + if (is_base64(orig_buf + idx + i - 3)) fromb64 += 4; -#endif + } + + } if ((o_pattern[i] ^ orig_buf[idx + i]) == xor_val[i] && xor_val[i]) { @@ -2229,45 +2494,70 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } #ifdef _DEBUG - fprintf(stderr, - "RTN idx=%u loop=%u xor=%u arith=%u tolower=%u toupper=%u " - "tohex=%u fromhex=%u to_0=%u to_slash=%u to_x=%u " - "from_0=%u from_slash=%u from_x=%u\n", - idx, i, xor, arith, tolower, toupper, tohex, fromhex, to_0, - to_slash, to_x, from_0, from_slash, from_x); - #ifdef CMPLOG_SOLVE_TRANSFORM_BASE64 - fprintf(stderr, "RTN idx=%u loop=%u tob64=%u from64=%u\n", tob64, - fromb64); - #endif + if (idx == 0) { + + fprintf(stderr, "RTN Z %s %s %s %s repl=%s\n", buf, pattern, orig_buf, + o_pattern, repl); + fprintf( + stderr, + "RTN Z idx=%u len=%u loop=%u xor=%u arith=%u tolower=%u toupper=%u " + "tohex=%u fromhex=%u to_0=%u to_slash=%u to_x=%u " + "from_0=%u from_slash=%u from_x=%u\n", + idx, its_len, i, xor, arith, tolower, toupper, tohex, fromhex, to_0, + to_slash, to_x, from_0, from_slash, from_x); + if (afl->cmplog_enable_xtreme_transform) { + + fprintf(stderr, "RTN Z idx=%u loop=%u tob64=%u from64=%u\n", idx, i, + tob64, fromb64); + + } + + } + #endif -#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64 - // input is base64 and converted to binary? convert repl to base64! - if ((i % 4) == 3 && i < 24 && fromb64 > i) { + if (afl->cmplog_enable_xtreme_transform) { - to_base64(repl, tmp, i + 1); - memcpy(buf + idx, tmp, i + 1); - if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } - // fprintf(stderr, "RTN ATTEMPT fromb64 %u result %u\n", fromb64, - // *status); + // input is base64 and converted to binary? convert repl to base64! + // fprintf(stderr, "FROMB64 idx=%u i=%u %% 4 == 3 && i < 24 && + // fromb64=%u > i, repl=%s\n", saved_idx, i, fromb64, repl); + if ((i % 4) == 3 && i < 24 && fromb64 > i) { - } + for (u32 hlen = i; hlen + saved_idx < len && hlen <= its_len; + ++hlen) { - // input is converted to base64? decode repl with base64! - if ((i % 3) == 2 && i < 24 && tob64 > i) { + u32 res = to_base64(repl, tmp, hlen); + // fprintf(stderr, "FROMB64 GOGO! idx=%u repl=%s tmp[%u]=%s + // hlen=%u\n", saved_idx, repl, res, tmp, hlen); + if (res + saved_idx < len) { - u32 olen = from_base64(repl, tmp, i + 1); - memcpy(buf + idx, tmp, olen); - if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } - // fprintf(stderr, "RTN ATTEMPT tob64 %u idx=%u result %u\n", tob64, - // idx, *status); + memcpy(buf + idx, tmp, res); + if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } + // fprintf(stderr, "RTN ATTEMPT FROMB64 idx=%u fromb64 %u %s %s + // result %u\n", saved_idx, fromb64, tmp, repl, + // *status); - } + } -#endif + } + + } + + // input is converted to base64? decode repl with base64! + if ((i % 3) == 2 && i < 24 && tob64 > i) { + + u32 olen = from_base64(repl, tmp, i + 1); + memcpy(buf + idx, tmp, olen); + if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } + // fprintf(stderr, "RTN ATTEMPT tob64 %u idx=%u result %u\n", tob64, + // idx, *status); + + } + + } // input is converted to hex? convert repl to binary! - if (i < 16 && tohex > i) { + if (afl->cmplog_enable_xtreme_transform && i < 16 && tohex > i) { u32 off; if (to_slash + to_x + to_0 == 2) { @@ -2292,8 +2582,8 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } // input is hex and converted to binary? convert repl to hex! - if (i && (i % 2) && i < 16 && fromhex && - fromhex + from_slash + from_x + from_0 > i) { + if (afl->cmplog_enable_xtreme_transform && (i % 2) == 1 && i < 16 && + fromhex && fromhex + from_slash + from_x + from_0 > i) { u8 off = 0; if (from_slash && from_x) { @@ -2328,31 +2618,37 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - if (to_up == 1) { + for (u32 hlen = i; hlen <= (i << 1) && hlen + idx < len; hlen += i) { - for (j = 0; j <= (i >> 1); j++) { + if (to_up == 1) { - tmp[off + (j << 1)] = hex_table_up[repl[j] >> 4]; - tmp[off + (j << 1) + 1] = hex_table_up[repl[j] % 16]; + for (j = 0; j <= (hlen >> 1); j++) { - } + tmp[off + (j << 1)] = hex_table_up[repl[j] >> 4]; + tmp[off + (j << 1) + 1] = hex_table_up[repl[j] % 16]; - } else { + } + + } else { - for (j = 0; j <= (i >> 1); j++) { + for (j = 0; j <= (hlen >> 1); j++) { - tmp[off + (j << 1)] = hex_table_low[repl[j] >> 4]; - tmp[off + (j << 1) + 1] = hex_table_low[repl[j] % 16]; + tmp[off + (j << 1)] = hex_table_low[repl[j] >> 4]; + tmp[off + (j << 1) + 1] = hex_table_low[repl[j] % 16]; + + } } - } + u32 tmp_l = hlen + 1 + off; + memcpy(buf + idx, tmp, tmp_l); + if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } + tmp[tmp_l] = 0; + // fprintf(stderr, "RTN ATTEMPT idx=%u len=%u fromhex %u %s %s result + // %u\n", idx, len, fromhex, tmp, repl, *status); + memcpy(buf + idx, save, tmp_l); - memcpy(buf + idx, tmp, i + 1 + off); - if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } - // fprintf(stderr, "RTN ATTEMPT fromhex %u result %u\n", fromhex, - // *status); - memcpy(buf + idx, save, i + 1 + off); + } } @@ -2401,11 +2697,9 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, if ((i >= 7 && (i >= xor&&i >= arith &&i >= tolower &&i >= toupper &&i > tohex &&i > - (fromhex + from_0 + from_x + from_slash + 1) -#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64 - && i > tob64 + 3 && i > fromb64 + 4 -#endif - )) || + (fromhex + from_0 + from_x + from_slash + 1) && + (afl->cmplog_enable_xtreme_transform && i > tob64 + 3 && + i > fromb64 + 4))) || repl[i] != changed_val[i] || *status == 1) { break; @@ -2418,8 +2712,6 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - //#endif - return 0; } @@ -2429,11 +2721,13 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, struct tainted *t; struct cmp_header *h = &afl->shm.cmp_map->headers[key]; - u32 i, j, idx, have_taint = 1, taint_len, loggeds; + u32 i, idx, have_taint = 1, taint_len, loggeds; u8 status = 0, found_one = 0; hshape = SHAPE_BYTES(h->shape); + if (hshape < 2) { return 0; } + if (h->hits > CMP_MAP_RTN_H) { loggeds = CMP_MAP_RTN_H; @@ -2452,36 +2746,52 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, struct cmpfn_operands *orig_o = &((struct cmpfn_operands *)afl->orig_cmp_map->log[key])[i]; - // opt not in the paper - for (j = 0; j < i; ++j) { + /* + // opt not in the paper + for (j = 0; j < i; ++j) { - if (!memcmp(&((struct cmpfn_operands *)afl->shm.cmp_map->log[key])[j], o, - sizeof(struct cmpfn_operands))) { + if (!memcmp(&((struct cmpfn_operands *)afl->shm.cmp_map->log[key])[j], + o, sizeof(struct cmpfn_operands))) { - goto rtn_fuzz_next_iter; + goto rtn_fuzz_next_iter; - } + } - } + } - /* + */ + +#ifdef _DEBUG + u32 j; struct cmp_header *hh = &afl->orig_cmp_map->headers[key]; - fprintf(stderr, "RTN N hits=%u id=%u shape=%u attr=%u v0=", h->hits, h->id, - hshape, h->attribute); + fprintf(stderr, "RTN N hits=%u shape=%u attr=%u v0=", h->hits, hshape, + h->attribute); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v0[j]); fprintf(stderr, " v1="); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v1[j]); - fprintf(stderr, "\nRTN O hits=%u id=%u shape=%u attr=%u o0=", hh->hits, - hh->id, hshape, hh->attribute); + fprintf(stderr, "\nRTN O hits=%u shape=%u attr=%u o0=", hh->hits, hshape, + hh->attribute); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v0[j]); fprintf(stderr, " o1="); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v1[j]); fprintf(stderr, "\n"); - */ +#endif + +#ifdef USE_HASHMAP + if (hshape <= 8 && hashmap_search_and_add_ptr(hshape - 1, o->v0) && + hashmap_search_and_add_ptr(hshape - 1, orig_o->v0) && + hashmap_search_and_add_ptr(hshape - 1, o->v1) && + hashmap_search_and_add_ptr(hshape - 1, orig_o->v1)) { + + continue; + + } + +#endif t = taint; while (t->next) { @@ -2515,7 +2825,7 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, status = 0; #ifdef _DEBUG - int w; + u32 w; fprintf(stderr, "key=%u idx=%u len=%u o0=", key, idx, hshape); for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", orig_o->v0[w]); @@ -2592,6 +2902,8 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, // shape_len), check_if_text_buf((u8 *)&o->v1, shape_len), v0_len, // o->v0, v1_len, o->v1); + // Note that this check differs from the line 1901, for RTN we are more + // opportunistic for adding to the dictionary than cmps if (!memcmp(o->v0, orig_o->v0, v0_len) || (!found_one || check_if_text_buf((u8 *)&o->v0, v0_len) == v0_len)) maybe_add_auto(afl, o->v0, v0_len); @@ -2603,7 +2915,7 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } - rtn_fuzz_next_iter: + // rtn_fuzz_next_iter: afl->stage_cur++; } @@ -2747,6 +3059,10 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { // Start insertion loop +#ifdef USE_HASHMAP + hashmap_reset(); +#endif + u64 orig_hit_cnt, new_hit_cnt; u64 orig_execs = afl->fsrv.total_execs; orig_hit_cnt = afl->queued_items + afl->saved_crashes; @@ -2816,12 +3132,7 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { } - } else if ((lvl & LVL1) - - //#ifdef CMPLOG_SOLVE_TRANSFORM - || ((lvl & LVL3) && afl->cmplog_enable_transform) - //#endif - ) { + } else if ((lvl & LVL1) || ((lvl & LVL3) && afl->cmplog_enable_transform)) { if (unlikely(rtn_fuzz(afl, k, orig_buf, buf, cbuf, len, lvl, taint))) { diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index ac4fb4a9..b62db1ea 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -5,12 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> and Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -60,6 +60,23 @@ fuzz_run_target(afl_state_t *afl, afl_forkserver_t *fsrv, u32 timeout) { fsrv_run_result_t res = afl_fsrv_run_target(fsrv, timeout, &afl->stop_soon); + /* If post_run() function is defined in custom mutator, the function will be + called each time after AFL++ executes the target program. */ + + if (unlikely(afl->custom_mutators_count)) { + + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (unlikely(el->afl_custom_post_run)) { + + el->afl_custom_post_run(el->data); + + } + + }); + + } + #ifdef PROFILING clock_gettime(CLOCK_REALTIME, &spec); time_spent_start = (spec.tv_sec * 1000000000) + spec.tv_nsec; @@ -152,20 +169,16 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { } - if (unlikely(afl->custom_mutators_count)) { - - LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { - - if (el->afl_custom_fuzz_send) { + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { - el->afl_custom_fuzz_send(el->data, *mem, new_size); - sent = 1; + if (el->afl_custom_fuzz_send) { - } + el->afl_custom_fuzz_send(el->data, *mem, new_size); + sent = 1; - }); + } - } + }); if (likely(!sent)) { @@ -186,7 +199,7 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { } - } else { + } else { /* !afl->custom_mutators_count */ if (unlikely(len < afl->min_length && !fix)) { @@ -198,27 +211,8 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { } - if (unlikely(afl->custom_mutators_count)) { - - LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { - - if (el->afl_custom_fuzz_send) { - - el->afl_custom_fuzz_send(el->data, *mem, len); - sent = 1; - - } - - }); - - } - - if (likely(!sent)) { - - /* boring uncustom. */ - afl_fsrv_write_to_testcase(&afl->fsrv, *mem, len); - - } + /* boring uncustom. */ + afl_fsrv_write_to_testcase(&afl->fsrv, *mem, len); } @@ -415,6 +409,7 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, u32 use_tmout = afl->fsrv.exec_tmout; u8 *old_sn = afl->stage_name; + u64 calibration_start_us = get_cur_time_us(); if (unlikely(afl->shm.cmplog_mode)) { q->exec_cksum = 0; } /* Be a bit more generous about timeouts when resuming sessions, or when @@ -510,6 +505,10 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, fault = fuzz_run_target(afl, &afl->fsrv, use_tmout); + // update the time spend in calibration after each execution, as those may + // be slow + update_calibration_time(afl, &calibration_start_us); + /* afl->stop_soon is set by the handler for Ctrl+C. When it's pressed, we want to bail out quickly. */ @@ -607,6 +606,8 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, } q->exec_us = diff_us / afl->stage_max; + if (unlikely(!q->exec_us)) { q->exec_us = 1; } + q->bitmap_size = count_bytes(afl, afl->fsrv.trace_bits); q->handicap = handicap; q->cal_failed = 0; @@ -656,6 +657,7 @@ abort_calibration: if (!first_run) { show_stats(afl); } + update_calibration_time(afl, &calibration_start_us); return fault; } @@ -675,11 +677,16 @@ void sync_fuzzers(afl_state_t *afl) { afl->stage_max = afl->stage_cur = 0; afl->cur_depth = 0; + u64 sync_start_us = get_cur_time_us(); /* Look at the entries created for every other fuzzer in the sync directory. */ while ((sd_ent = readdir(sd))) { + // since sync can take substantial amounts of time, update time spend every + // iteration + update_sync_time(afl, &sync_start_us); + u8 qd_synced_path[PATH_MAX], qd_path[PATH_MAX]; u32 min_accept = 0, next_min_accept = 0; @@ -766,6 +773,8 @@ void sync_fuzzers(afl_state_t *afl) { afl->stage_cur = 0; afl->stage_max = 0; + show_stats(afl); + /* For every file queued by this fuzzer, parse ID and see if we have looked at it before; exec a test case if not. */ @@ -817,15 +826,15 @@ void sync_fuzzers(afl_state_t *afl) { /* See what happens. We rely on save_if_interesting() to catch major errors and save the test case. */ - (void)write_to_testcase(afl, (void **)&mem, st.st_size, 1); + u32 new_len = write_to_testcase(afl, (void **)&mem, st.st_size, 1); fault = fuzz_run_target(afl, &afl->fsrv, afl->fsrv.exec_tmout); if (afl->stop_soon) { goto close_sync; } afl->syncing_party = sd_ent->d_name; - afl->queued_imported += - save_if_interesting(afl, mem, st.st_size, fault); + afl->queued_imported += save_if_interesting(afl, mem, new_len, fault); + show_stats(afl); afl->syncing_party = 0; munmap(mem, st.st_size); @@ -867,6 +876,9 @@ void sync_fuzzers(afl_state_t *afl) { if (afl->foreign_sync_cnt) read_foreign_testcases(afl, 0); + // add time in sync one last time + update_sync_time(afl, &sync_start_us); + afl->last_sync_time = get_cur_time(); afl->last_sync_cycle = afl->queue_cycle; @@ -878,8 +890,9 @@ void sync_fuzzers(afl_state_t *afl) { u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { + u8 needs_write = 0, fault = 0; u32 orig_len = q->len; - + u64 trim_start_us = get_cur_time_us(); /* Custom mutator trimmer */ if (afl->custom_mutators_count) { @@ -903,11 +916,15 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { } - if (custom_trimmed) return trimmed_case; + if (custom_trimmed) { + + fault = trimmed_case; + goto abort_trimming; + + } } - u8 needs_write = 0, fault = 0; u32 trim_exec = 0; u32 remove_len; u32 len_p2; @@ -918,7 +935,12 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { detected, it will still work to some extent, so we don't check for this. */ - if (q->len < 5) { return 0; } + if (unlikely(q->len < 5)) { + + fault = 0; + goto abort_trimming; + + } afl->stage_name = afl->stage_name_buf; afl->bytes_trim_in += q->len; @@ -952,6 +974,8 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { fault = fuzz_run_target(afl, &afl->fsrv, afl->fsrv.exec_tmout); + update_trim_time(afl, &trim_start_us); + if (afl->stop_soon || fault == FSRV_RUN_ERROR) { goto abort_trimming; } /* Note that we don't keep track of crashes or hangs here; maybe TODO? @@ -978,7 +1002,6 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { /* Let's save a clean trace, which will be needed by update_bitmap_score once we're done with the trimming stuff. */ - if (!needs_write) { needs_write = 1; @@ -993,7 +1016,6 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { } /* Since this can be slow, update the screen every now and then. */ - if (!(trim_exec++ % afl->stats_update_freq)) { show_stats(afl); } ++afl->stage_cur; @@ -1008,6 +1030,68 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { if (needs_write) { + // run afl_custom_post_process + + if (unlikely(afl->custom_mutators_count) && + likely(!afl->afl_env.afl_post_process_keep_original)) { + + ssize_t new_size = q->len; + u8 *new_mem = in_buf; + u8 *new_buf = NULL; + + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_post_process) { + + new_size = el->afl_custom_post_process(el->data, new_mem, new_size, + &new_buf); + + if (unlikely(!new_buf || new_size <= 0)) { + + new_size = 0; + new_buf = new_mem; + + } else { + + new_mem = new_buf; + + } + + } + + }); + + if (unlikely(!new_size)) { + + new_size = q->len; + new_mem = in_buf; + + } + + if (unlikely(new_size < afl->min_length)) { + + new_size = afl->min_length; + + } else if (unlikely(new_size > afl->max_length)) { + + new_size = afl->max_length; + + } + + q->len = new_size; + + if (new_mem != in_buf && new_mem != NULL) { + + new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), new_size); + if (unlikely(!new_buf)) { PFATAL("alloc"); } + memcpy(new_buf, new_mem, new_size); + + in_buf = new_buf; + + } + + } + s32 fd; if (unlikely(afl->no_unlink)) { @@ -1045,8 +1129,9 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { } abort_trimming: - afl->bytes_trim_out += q->len; + update_trim_time(afl, &trim_start_us); + return fault; } @@ -1110,4 +1195,3 @@ common_fuzz_stuff(afl_state_t *afl, u8 *out_buf, u32 len) { return 0; } - diff --git a/src/afl-fuzz-skipdet.c b/src/afl-fuzz-skipdet.c new file mode 100644 index 00000000..e52d59a3 --- /dev/null +++ b/src/afl-fuzz-skipdet.c @@ -0,0 +1,403 @@ + + +#include "afl-fuzz.h" + +void flip_range(u8 *input, u32 pos, u32 size) { + + for (u32 i = 0; i < size; i++) + input[pos + i] ^= 0xFF; + + return; + +} + +#define MAX_EFF_TIMEOUT (10 * 60 * 1000) +#define MAX_DET_TIMEOUT (15 * 60 * 1000) +u8 is_det_timeout(u64 cur_ms, u8 is_flip) { + + if (is_flip) { + + if (unlikely(get_cur_time() - cur_ms > MAX_EFF_TIMEOUT)) return 1; + + } else { + + if (unlikely(get_cur_time() - cur_ms > MAX_DET_TIMEOUT)) return 1; + + } + + return 0; + +} + +/* decide if the seed should be deterministically fuzzed */ + +u8 should_det_fuzz(afl_state_t *afl, struct queue_entry *q) { + + if (!afl->skipdet_g->virgin_det_bits) { + + afl->skipdet_g->virgin_det_bits = + (u8 *)ck_alloc(sizeof(u8) * afl->fsrv.map_size); + + } + + if (!q->favored || q->passed_det) return 0; + if (!q->trace_mini) return 0; + + if (!afl->skipdet_g->last_cov_undet) + afl->skipdet_g->last_cov_undet = get_cur_time(); + + if (get_cur_time() - afl->skipdet_g->last_cov_undet >= THRESHOLD_DEC_TIME) { + + if (afl->skipdet_g->undet_bits_threshold >= 2) { + + afl->skipdet_g->undet_bits_threshold *= 0.75; + afl->skipdet_g->last_cov_undet = get_cur_time(); + + } + + } + + u32 new_det_bits = 0; + + for (u32 i = 0; i < afl->fsrv.map_size; i++) { + + if (unlikely(q->trace_mini[i >> 3] & (1 << (i & 7)))) { + + if (!afl->skipdet_g->virgin_det_bits[i]) { new_det_bits++; } + + } + + } + + if (!afl->skipdet_g->undet_bits_threshold) + afl->skipdet_g->undet_bits_threshold = new_det_bits * 0.05; + + if (new_det_bits >= afl->skipdet_g->undet_bits_threshold) { + + afl->skipdet_g->last_cov_undet = get_cur_time(); + q->skipdet_e->undet_bits = new_det_bits; + + for (u32 i = 0; i < afl->fsrv.map_size; i++) { + + if (unlikely(q->trace_mini[i >> 3] & (1 << (i & 7)))) { + + if (!afl->skipdet_g->virgin_det_bits[i]) + afl->skipdet_g->virgin_det_bits[i] = 1; + + } + + } + + return 1; + + } + + return 0; + +} + +/* + consists of two stages that + return 0 if exec failed. +*/ + +u8 skip_deterministic_stage(afl_state_t *afl, u8 *orig_buf, u8 *out_buf, + u32 len, u64 before_det_time) { + + u64 orig_hit_cnt, new_hit_cnt; + + if (afl->queue_cur->skipdet_e->done_eff) return 1; + + if (!should_det_fuzz(afl, afl->queue_cur)) return 1; + + /* Add check to make sure that for seeds without too much undet bits, + we ignore them */ + + /****************** + * SKIP INFERENCE * + ******************/ + + afl->stage_short = "inf"; + afl->stage_name = "inference"; + afl->stage_cur = 0; + orig_hit_cnt = afl->queued_items + afl->saved_crashes; + + u8 *inf_eff_map = (u8 *)ck_alloc(sizeof(u8) * len); + memset(inf_eff_map, 1, sizeof(u8) * len); + + if (common_fuzz_stuff(afl, orig_buf, len)) { return 0; } + + u64 prev_cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); + u64 _prev_cksum = prev_cksum; + + if (MINIMAL_BLOCK_SIZE * 8 < len) { + + // u64 size_skiped = 0, quick_skip_exec = total_execs, quick_skip_time = + // get_cur_time(); + u64 pre_inf_exec = afl->fsrv.total_execs, pre_inf_time = get_cur_time(); + + /* if determine stage time / input size is too small, just go ahead */ + + u32 pos = 0, cur_block_size = MINIMAL_BLOCK_SIZE, max_block_size = len / 8; + + while (pos < len - 1) { + + cur_block_size = MINIMAL_BLOCK_SIZE; + + while (cur_block_size < max_block_size) { + + u32 flip_block_size = + (cur_block_size + pos < len) ? cur_block_size : len - 1 - pos; + + afl->stage_cur += 1; + + flip_range(out_buf, pos, flip_block_size); + + if (common_fuzz_stuff(afl, out_buf, len)) return 0; + + flip_range(out_buf, pos, flip_block_size); + + u64 cksum = + hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); + + // printf("Now trying range %d with %d, %s.\n", pos, cur_block_size, + // (cksum == prev_cksum) ? (u8*)"Yes" : (u8*) "Not"); + + /* continue until we fail or exceed length */ + if (cksum == _prev_cksum) { + + cur_block_size *= 2; + + if (cur_block_size >= len - 1 - pos) break; + + } else { + + break; + + } + + } + + if (cur_block_size == MINIMAL_BLOCK_SIZE) { + + /* we failed early on*/ + + pos += cur_block_size; + + } else { + + u32 cur_skip_len = (cur_block_size / 2 + pos < len) + ? (cur_block_size / 2) + : (len - pos - 1); + + memset(inf_eff_map + pos, 0, cur_skip_len); + + afl->skipdet_g->inf_prof->inf_skipped_bytes += cur_skip_len; + + pos += cur_skip_len; + + } + + } + + afl->skipdet_g->inf_prof->inf_execs_cost += + (afl->fsrv.total_execs - pre_inf_exec); + afl->skipdet_g->inf_prof->inf_time_cost += (get_cur_time() - pre_inf_time); + // PFATAL("Done, now have %d bytes skipped, with exec %lld, time %lld.\n", + // afl->inf_skipped_bytes, afl->inf_execs_cost, afl->inf_time_cost); + + } else + + memset(inf_eff_map, 1, len); + + new_hit_cnt = afl->queued_items + afl->saved_crashes; + + afl->stage_finds[STAGE_INF] += new_hit_cnt - orig_hit_cnt; + afl->stage_cycles[STAGE_INF] += afl->stage_cur; + + /**************************** + * Quick Skip Effective Map * + ****************************/ + + /* Quick Effective Map Calculation */ + + afl->stage_short = "quick"; + afl->stage_name = "quick eff"; + afl->stage_cur = 0; + afl->stage_max = 32 * 1024; + + orig_hit_cnt = afl->queued_items + afl->saved_crashes; + + u32 before_skip_inf = afl->queued_items; + + /* clean all the eff bytes, since previous eff bytes are already fuzzed */ + u8 *skip_eff_map = afl->queue_cur->skipdet_e->skip_eff_map, + *done_inf_map = afl->queue_cur->skipdet_e->done_inf_map; + + if (!skip_eff_map) { + + skip_eff_map = (u8 *)ck_alloc(sizeof(u8) * len); + afl->queue_cur->skipdet_e->skip_eff_map = skip_eff_map; + + } else { + + memset(skip_eff_map, 0, sizeof(u8) * len); + + } + + /* restore the starting point */ + if (!done_inf_map) { + + done_inf_map = (u8 *)ck_alloc(sizeof(u8) * len); + afl->queue_cur->skipdet_e->done_inf_map = done_inf_map; + + } else { + + for (afl->stage_cur = 0; afl->stage_cur < len; afl->stage_cur++) { + + if (done_inf_map[afl->stage_cur] == 0) break; + + } + + } + + /* depending on the seed's performance, we could search eff bytes + for multiple rounds */ + + u8 eff_round_continue = 1, eff_round_done = 0, done_eff = 0, repeat_eff = 0, + fuzz_nearby = 0, *non_eff_bytes = 0; + + u64 before_eff_execs = afl->fsrv.total_execs; + + if (getenv("REPEAT_EFF")) repeat_eff = 1; + if (getenv("FUZZ_NEARBY")) fuzz_nearby = 1; + + if (fuzz_nearby) { + + non_eff_bytes = (u8 *)ck_alloc(sizeof(u8) * len); + + // clean exec cksum + if (common_fuzz_stuff(afl, out_buf, len)) { return 0; } + prev_cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); + + } + + do { + + eff_round_continue = 0; + afl->stage_max = 32 * 1024; + + for (; afl->stage_cur < afl->stage_max && afl->stage_cur < len; + ++afl->stage_cur) { + + afl->stage_cur_byte = afl->stage_cur; + + if (!inf_eff_map[afl->stage_cur_byte] || + skip_eff_map[afl->stage_cur_byte]) + continue; + + if (is_det_timeout(before_det_time, 1)) { goto cleanup_skipdet; } + + u8 orig = out_buf[afl->stage_cur_byte], replace = rand_below(afl, 256); + + while (replace == orig) { + + replace = rand_below(afl, 256); + + } + + out_buf[afl->stage_cur_byte] = replace; + + before_skip_inf = afl->queued_items; + + if (common_fuzz_stuff(afl, out_buf, len)) { return 0; } + + out_buf[afl->stage_cur_byte] = orig; + + if (fuzz_nearby) { + + if (prev_cksum == + hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST)) { + + non_eff_bytes[afl->stage_cur_byte] = 1; + + } + + } + + if (afl->queued_items != before_skip_inf) { + + skip_eff_map[afl->stage_cur_byte] = 1; + afl->queue_cur->skipdet_e->quick_eff_bytes += 1; + + if (afl->stage_max < MAXIMUM_QUICK_EFF_EXECS) { afl->stage_max *= 2; } + + if (afl->stage_max == MAXIMUM_QUICK_EFF_EXECS && repeat_eff) + eff_round_continue = 1; + + } + + done_inf_map[afl->stage_cur_byte] = 1; + + } + + afl->stage_cur = 0; + done_eff = 1; + + if (++eff_round_done >= 8) break; + + } while (eff_round_continue); + + new_hit_cnt = afl->queued_items + afl->saved_crashes; + + afl->stage_finds[STAGE_QUICK] += new_hit_cnt - orig_hit_cnt; + afl->stage_cycles[STAGE_QUICK] += (afl->fsrv.total_execs - before_eff_execs); + +cleanup_skipdet: + + if (fuzz_nearby) { + + u8 *nearby_bytes = (u8 *)ck_alloc(sizeof(u8) * len); + + u32 i = 3; + while (i < len) { + + // assume DWORD size, from i - 3 -> i + 3 + if (skip_eff_map[i]) { + + u32 fill_length = (i + 3 < len) ? 7 : len - i + 2; + memset(nearby_bytes + i - 3, 1, fill_length); + i += 3; + + } else + + i += 1; + + } + + for (i = 0; i < len; i++) { + + if (nearby_bytes[i] && !non_eff_bytes[i]) skip_eff_map[i] = 1; + + } + + ck_free(nearby_bytes); + ck_free(non_eff_bytes); + + } + + if (done_eff) { + + afl->queue_cur->skipdet_e->continue_inf = 0; + afl->queue_cur->skipdet_e->done_eff = 1; + + } else { + + afl->queue_cur->skipdet_e->continue_inf = 1; + + } + + return 1; + +} + diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 5e736029..333d57b2 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -28,10 +28,6 @@ #include "afl-fuzz.h" #include "envs.h" -s8 interesting_8[] = {INTERESTING_8}; -s16 interesting_16[] = {INTERESTING_8, INTERESTING_16}; -s32 interesting_32[] = {INTERESTING_8, INTERESTING_16, INTERESTING_32}; - char *power_names[POWER_SCHEDULES_NUM] = {"explore", "mmopt", "exploit", "fast", "coe", "lin", "quad", "rare", "seek"}; @@ -89,9 +85,8 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->w_end = 0.3; afl->g_max = 5000; afl->period_pilot_tmp = 5000.0; - afl->schedule = FAST; /* Power schedule (default: FAST) */ + afl->schedule = EXPLORE; /* Power schedule (default: EXPLORE)*/ afl->havoc_max_mult = HAVOC_MAX_MULT; - afl->clear_screen = 1; /* Window resized? */ afl->havoc_div = 1; /* Cycle count divisor for havoc */ afl->stage_name = "init"; /* Name of the current fuzz stage */ @@ -103,11 +98,12 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->stats_update_freq = 1; afl->stats_file_update_freq_msecs = STATS_UPDATE_SEC * 1000; afl->stats_avg_exec = 0; - afl->skip_deterministic = 1; + afl->skip_deterministic = 0; afl->sync_time = SYNC_TIME; afl->cmplog_lvl = 2; afl->min_length = 1; afl->max_length = MAX_FILE; + afl->switch_fuzz_mode = STRATEGY_SWITCH_TIME * 1000; #ifndef NO_SPLICING afl->use_splicing = 1; #endif @@ -140,6 +136,14 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->fsrv.child_pid = -1; afl->fsrv.out_dir_fd = -1; + /* Init SkipDet */ + afl->skipdet_g = + (struct skipdet_global *)ck_alloc(sizeof(struct skipdet_global)); + afl->skipdet_g->inf_prof = + (struct inf_profile *)ck_alloc(sizeof(struct inf_profile)); + afl->havoc_prof = + (struct havoc_profile *)ck_alloc(sizeof(struct havoc_profile)); + init_mopt_globals(afl); list_append(&afl_states, afl); @@ -199,6 +203,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_exit_on_time = (u8 *)get_afl_env(afl_environment_variables[i]); + } else if (!strncmp(env, "AFL_CRASHING_SEEDS_AS_NEW_CRASH", + + afl_environment_variable_len)) { + + afl->afl_env.afl_crashing_seeds_as_new_crash = + atoi((u8 *)get_afl_env(afl_environment_variables[i])); + } else if (!strncmp(env, "AFL_NO_AFFINITY", afl_environment_variable_len)) { @@ -261,6 +272,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_import_first = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_FINAL_SYNC", + + afl_environment_variable_len)) { + + afl->afl_env.afl_final_sync = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_CUSTOM_MUTATOR_ONLY", afl_environment_variable_len)) { @@ -275,6 +293,16 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_cmplog_only_new = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_DISABLE_REDUNDANT", + + afl_environment_variable_len) || + !strncmp(env, "AFL_NO_REDUNDANT", + + afl_environment_variable_len)) { + + afl->afl_env.afl_disable_redundant = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_NO_STARTUP_CALIBRATION", afl_environment_variable_len)) { @@ -301,6 +329,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_ignore_problems = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_IGNORE_SEED_PROBLEMS", + + afl_environment_variable_len)) { + + afl->afl_env.afl_ignore_seed_problems = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_IGNORE_TIMEOUTS", afl_environment_variable_len)) { @@ -594,6 +629,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { } + } else if (!strncmp(env, "AFL_SHA1_FILENAMES", + + afl_environment_variable_len)) { + + afl->afl_env.afl_sha1_filenames = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } } else { diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 07157bf7..ffe56cde 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -5,11 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and - Andrea Fioraldi <andreafioraldi@gmail.com> + Dominik Meier <mail@dmnk.co>, + Andrea Fioraldi <andreafioraldi@gmail.com>, and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,6 +28,50 @@ #include "envs.h" #include <limits.h> +static char fuzzing_state[4][12] = {"started :-)", "in progress", "final phase", + "finished..."}; + +char *get_fuzzing_state(afl_state_t *afl) { + + u64 cur_ms = get_cur_time(); + u64 last_find = cur_ms - afl->last_find_time; + u64 cur_run_time = cur_ms - afl->start_time; + u64 cur_total_run_time = afl->prev_run_time + cur_run_time; + + if (unlikely(afl->non_instrumented_mode)) { + + return fuzzing_state[1]; + + } else if (unlikely(cur_run_time < 60 * 3 * 1000 || + + cur_total_run_time < 60 * 5 * 1000)) { + + return fuzzing_state[0]; + + } else { + + u64 last_find_100 = 100 * last_find; + u64 percent_cur = last_find_100 / cur_run_time; + u64 percent_total = last_find_100 / cur_total_run_time; + + if (unlikely(percent_cur >= 80 && percent_total >= 80)) { + + return fuzzing_state[3]; + + } else if (unlikely(percent_cur >= 55 && percent_total >= 55)) { + + return fuzzing_state[2]; + + } else { + + return fuzzing_state[1]; + + } + + } + +} + /* Write fuzzer setup file */ void write_setup_file(afl_state_t *afl, u32 argc, char **argv) { @@ -89,6 +134,12 @@ void write_setup_file(afl_state_t *afl, u32 argc, char **argv) { } +static bool starts_with(char *key, char *line) { + + return strncmp(key, line, strlen(key)) == 0; + +} + /* load some of the existing stats file when resuming.*/ void load_stats_file(afl_state_t *afl) { @@ -131,58 +182,84 @@ void load_stats_file(afl_state_t *afl) { strcpy(keystring, lstartptr); lptr++; char *nptr; - switch (lineno) { - - case 3: - if (!strcmp(keystring, "run_time ")) - afl->prev_run_time = 1000 * strtoull(lptr, &nptr, 10); - break; - case 5: - if (!strcmp(keystring, "cycles_done ")) - afl->queue_cycle = - strtoull(lptr, &nptr, 10) ? strtoull(lptr, &nptr, 10) + 1 : 0; - break; - case 7: - if (!strcmp(keystring, "execs_done ")) - afl->fsrv.total_execs = strtoull(lptr, &nptr, 10); - break; - case 10: - if (!strcmp(keystring, "corpus_count ")) { - - u32 corpus_count = strtoul(lptr, &nptr, 10); - if (corpus_count != afl->queued_items) { - - WARNF( - "queue/ has been modified -- things might not work, you're " - "on your own!"); - - } - - } - - break; - case 12: - if (!strcmp(keystring, "corpus_found ")) - afl->queued_discovered = strtoul(lptr, &nptr, 10); - break; - case 13: - if (!strcmp(keystring, "corpus_imported ")) - afl->queued_imported = strtoul(lptr, &nptr, 10); - break; - case 14: - if (!strcmp(keystring, "max_depth ")) - afl->max_depth = strtoul(lptr, &nptr, 10); - break; - case 21: - if (!strcmp(keystring, "saved_crashes ")) - afl->saved_crashes = strtoull(lptr, &nptr, 10); - break; - case 22: - if (!strcmp(keystring, "saved_hangs ")) - afl->saved_hangs = strtoull(lptr, &nptr, 10); - break; - default: - break; + if (starts_with("run_time", keystring)) { + + afl->prev_run_time = 1000 * strtoull(lptr, &nptr, 10); + + } + + if (starts_with("cycles_done", keystring)) { + + afl->queue_cycle = + strtoull(lptr, &nptr, 10) ? strtoull(lptr, &nptr, 10) + 1 : 0; + + } + + if (starts_with("calibration_time", keystring)) { + + afl->calibration_time_us = strtoull(lptr, &nptr, 10) * 1000000; + + } + + if (starts_with("sync_time", keystring)) { + + afl->sync_time_us = strtoull(lptr, &nptr, 10) * 1000000; + + } + + if (starts_with("trim_time", keystring)) { + + afl->trim_time_us = strtoull(lptr, &nptr, 10) * 1000000; + + } + + if (starts_with("execs_done", keystring)) { + + afl->fsrv.total_execs = strtoull(lptr, &nptr, 10); + + } + + if (starts_with("corpus_count", keystring)) { + + u32 corpus_count = strtoul(lptr, &nptr, 10); + if (corpus_count != afl->queued_items) { + + WARNF( + "queue/ has been modified -- things might not work, you're " + "on your own!"); + sleep(3); + + } + + } + + if (starts_with("corpus_found", keystring)) { + + afl->queued_discovered = strtoul(lptr, &nptr, 10); + + } + + if (starts_with("corpus_imported", keystring)) { + + afl->queued_imported = strtoul(lptr, &nptr, 10); + + } + + if (starts_with("max_depth", keystring)) { + + afl->max_depth = strtoul(lptr, &nptr, 10); + + } + + if (starts_with("saved_crashes", keystring)) { + + afl->saved_crashes = strtoull(lptr, &nptr, 10); + + } + + if (starts_with("saved_hangs", keystring)) { + + afl->saved_hangs = strtoull(lptr, &nptr, 10); } @@ -206,11 +283,13 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, #endif u64 cur_time = get_cur_time(); - u8 fn[PATH_MAX]; + u8 fn_tmp[PATH_MAX]; + u8 fn_final[PATH_MAX]; FILE *f; - snprintf(fn, PATH_MAX, "%s/fuzzer_stats", afl->out_dir); - f = create_ffile(fn); + snprintf(fn_tmp, PATH_MAX, "%s/.fuzzer_stats_tmp", afl->out_dir); + snprintf(fn_final, PATH_MAX, "%s/fuzzer_stats", afl->out_dir); + f = create_ffile(fn_tmp); /* Keep last values in case we're called from another context where exec/sec stats and such are not readily available. */ @@ -242,6 +321,9 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, #ifndef __HAIKU__ if (getrusage(RUSAGE_CHILDREN, &rus)) { rus.ru_maxrss = 0; } #endif + u64 runtime_ms = afl->prev_run_time + cur_time - afl->start_time; + u64 overhead_ms = (afl->calibration_time_us + afl->sync_time_us + afl->trim_time_us) / 1000; + if (!runtime_ms) { runtime_ms = 1; } fprintf( f, @@ -252,6 +334,10 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, "cycles_done : %llu\n" "cycles_wo_finds : %llu\n" "time_wo_finds : %llu\n" + "fuzz_time : %llu\n" + "calibration_time : %llu\n" + "sync_time : %llu\n" + "trim_time : %llu\n" "execs_done : %llu\n" "execs_per_sec : %0.02f\n" "execs_ps_last_min : %0.02f\n" @@ -289,18 +375,18 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, "\n" "target_mode : %s%s%s%s%s%s%s%s%s%s\n" "command_line : %s\n", - (afl->start_time - afl->prev_run_time) / 1000, cur_time / 1000, - (afl->prev_run_time + cur_time - afl->start_time) / 1000, (u32)getpid(), + (afl->start_time /*- afl->prev_run_time*/) / 1000, cur_time / 1000, + runtime_ms / 1000, (u32)getpid(), afl->queue_cycle ? (afl->queue_cycle - 1) : 0, afl->cycles_wo_finds, afl->longest_find_time > cur_time - afl->last_find_time ? afl->longest_find_time / 1000 : ((afl->start_time == 0 || afl->last_find_time == 0) ? 0 : (cur_time - afl->last_find_time) / 1000), - afl->fsrv.total_execs, - afl->fsrv.total_execs / - ((double)(afl->prev_run_time + get_cur_time() - afl->start_time) / - 1000), + (runtime_ms - MIN(runtime_ms, overhead_ms)) / 1000, + afl->calibration_time_us / 1000000, afl->sync_time_us / 1000000, + afl->trim_time_us / 1000000, afl->fsrv.total_execs, + afl->fsrv.total_execs / ((double)(runtime_ms) / 1000), afl->last_avg_execs_saved, afl->queued_items, afl->queued_favored, afl->queued_discovered, afl->queued_imported, afl->queued_variable, afl->max_depth, afl->current_entry, afl->pending_favored, @@ -368,6 +454,7 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, } fclose(f); + rename(fn_tmp, fn_final); } @@ -456,6 +543,44 @@ void maybe_update_plot_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, } +/* Log deterministic stage efficiency */ + +void plot_profile_data(afl_state_t *afl, struct queue_entry *q) { + + u64 current_ms = get_cur_time() - afl->start_time; + + u32 current_edges = count_non_255_bytes(afl, afl->virgin_bits); + double det_finding_rate = (double)afl->havoc_prof->total_det_edge * 100.0 / + (double)current_edges, + det_time_rate = (double)afl->havoc_prof->total_det_time * 100.0 / + (double)current_ms; + + u32 ndet_bits = 0; + for (u32 i = 0; i < afl->fsrv.map_size; i++) { + + if (afl->skipdet_g->virgin_det_bits[i]) ndet_bits += 1; + + } + + double det_fuzzed_rate = (double)ndet_bits * 100.0 / (double)current_edges; + + fprintf(afl->fsrv.det_plot_file, + "[%02lld:%02lld:%02lld] fuzz %d (%d), find %d/%d among %d(%02.2f) " + "and spend %lld/%lld(%02.2f), cover %02.2f yet, %d/%d undet bits, " + "continue %d.\n", + current_ms / 1000 / 3600, (current_ms / 1000 / 60) % 60, + (current_ms / 1000) % 60, afl->current_entry, q->fuzz_level, + afl->havoc_prof->edge_det_stage, afl->havoc_prof->edge_havoc_stage, + current_edges, det_finding_rate, + afl->havoc_prof->det_stage_time / 1000, + afl->havoc_prof->havoc_stage_time / 1000, det_time_rate, + det_fuzzed_rate, q->skipdet_e->undet_bits, + afl->skipdet_g->undet_bits_threshold, q->skipdet_e->continue_inf); + + fflush(afl->fsrv.det_plot_file); + +} + /* Check terminal dimensions after resize. */ static void check_term_size(afl_state_t *afl) { @@ -505,9 +630,9 @@ void show_stats_normal(afl_state_t *afl) { cur_ms = get_cur_time(); - if (afl->most_time_key) { + if (afl->most_time_key && afl->queue_cycle) { - if (afl->most_time * 1000 < cur_ms - afl->start_time) { + if (afl->most_time * 1000 + afl->sync_time_us / 1000 < cur_ms - afl->start_time) { afl->most_time_key = 2; afl->stop_soon = 2; @@ -516,7 +641,7 @@ void show_stats_normal(afl_state_t *afl) { } - if (afl->most_execs_key == 1) { + if (afl->most_execs_key == 1 && afl->queue_cycle) { if (afl->most_execs <= afl->fsrv.total_execs) { @@ -734,10 +859,29 @@ void show_stats_normal(afl_state_t *afl) { if (unlikely(!banner[0])) { char *si = ""; + char *fuzzer_name; + if (afl->sync_id) { si = afl->sync_id; } memset(banner, 0, sizeof(banner)); - banner_len = (afl->crash_mode ? 20 : 18) + strlen(VERSION) + strlen(si) + - strlen(afl->power_name) + 4 + 6; + + banner_len = strlen(VERSION) + strlen(si) + strlen(afl->power_name) + 4 + 6; + + if (afl->crash_mode) { + + fuzzer_name = "peruvian were-rabbit"; + + } else { + + fuzzer_name = "american fuzzy lop"; + if (banner_len + strlen(fuzzer_name) + strlen(afl->use_banner) > 75) { + + fuzzer_name = "AFL"; + + } + + } + + banner_len += strlen(fuzzer_name); if (strlen(afl->use_banner) + banner_len > 75) { @@ -754,19 +898,18 @@ void show_stats_normal(afl_state_t *afl) { if (afl->fsrv.nyx_mode) { snprintf(banner + banner_pad, sizeof(banner) - banner_pad, - "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s] - Nyx", - afl->crash_mode ? cPIN "peruvian were-rabbit" - : cYEL "american fuzzy lop", - si, afl->use_banner, afl->power_name); + "%s%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN + "[%s] - Nyx", + afl->crash_mode ? cPIN : cYEL, fuzzer_name, si, afl->use_banner, + afl->power_name); } else { #endif snprintf(banner + banner_pad, sizeof(banner) - banner_pad, - "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s]", - afl->crash_mode ? cPIN "peruvian were-rabbit" - : cYEL "american fuzzy lop", - si, afl->use_banner, afl->power_name); + "%s%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s]", + afl->crash_mode ? cPIN : cYEL, fuzzer_name, si, afl->use_banner, + afl->power_name); #ifdef __linux__ @@ -774,6 +917,10 @@ void show_stats_normal(afl_state_t *afl) { #endif + if (banner_pad) + for (u32 i = 0; i < banner_pad; ++i) + strcat(banner, " "); + } SAYF("\n%s\n", banner); @@ -995,7 +1142,7 @@ void show_stats_normal(afl_state_t *afl) { sprintf(tmp, "%s (%s%s saved)", u_stringify_int(IB(0), afl->total_tmouts), u_stringify_int(IB(1), afl->saved_tmouts), - (afl->saved_hangs >= KEEP_UNIQUE_HANG) ? "+" : ""); + (afl->saved_tmouts >= KEEP_UNIQUE_HANG) ? "+" : ""); SAYF(bSTG bV bSTOP " total tmouts : " cRST "%-20s" bSTG bV "\n", tmp); @@ -1010,7 +1157,7 @@ void show_stats_normal(afl_state_t *afl) { } else if (likely(afl->skip_deterministic)) { - strcpy(tmp, "disabled (default, enable with -D)"); + strcpy(tmp, "disabled (-z switch used)"); } else { @@ -1282,7 +1429,11 @@ void show_stats_normal(afl_state_t *afl) { } /* Last line */ - SAYF(SET_G1 "\n" bSTG bLB bH30 bH20 bH2 bRB bSTOP cRST RESET_G1); + + SAYF(SET_G1 "\n" bSTG bLB bH cCYA bSTOP " strategy:" cPIN + " %s " bSTG bH10 cCYA bSTOP " state:" cPIN + " %s " bSTG bH2 bRB bSTOP cRST RESET_G1, + afl->fuzz_mode == 0 ? "explore" : "exploit", get_fuzzing_state(afl)); #undef IB @@ -1309,9 +1460,9 @@ void show_stats_pizza(afl_state_t *afl) { cur_ms = get_cur_time(); - if (afl->most_time_key) { + if (afl->most_time_key && afl->queue_cycle) { - if (afl->most_time * 1000 < cur_ms - afl->start_time) { + if (afl->most_time * 1000 + afl->sync_time_us / 1000 < cur_ms - afl->start_time) { afl->most_time_key = 2; afl->stop_soon = 2; @@ -1320,7 +1471,7 @@ void show_stats_pizza(afl_state_t *afl) { } - if (afl->most_execs_key == 1) { + if (afl->most_execs_key == 1 && afl->queue_cycle) { if (afl->most_execs <= afl->fsrv.total_execs) { @@ -1823,7 +1974,7 @@ void show_stats_pizza(afl_state_t *afl) { sprintf(tmp, "%s (%s%s saved)", u_stringify_int(IB(0), afl->total_tmouts), u_stringify_int(IB(1), afl->saved_tmouts), - (afl->saved_hangs >= KEEP_UNIQUE_HANG) ? "+" : ""); + (afl->saved_tmouts >= KEEP_UNIQUE_HANG) ? "+" : ""); SAYF(bSTG bV bSTOP " burned pizzas : " cRST "%-20s" bSTG bV "\n", @@ -2260,7 +2411,12 @@ void show_init_stats(afl_state_t *afl) { stringify_int(IB(0), min_us), stringify_int(IB(1), max_us), stringify_int(IB(2), avg_us)); - if (afl->timeout_given != 1) { + if (afl->timeout_given == 3) { + + ACTF("Applying timeout settings from resumed session (%u ms).", + afl->fsrv.exec_tmout); + + } else if (afl->timeout_given != 1) { /* Figure out the appropriate timeout. The basic idea is: 5x average or 1x max, rounded up to EXEC_TM_ROUND ms and capped at 1 second. @@ -2302,11 +2458,6 @@ void show_init_stats(afl_state_t *afl) { afl->timeout_given = 1; - } else if (afl->timeout_given == 3) { - - ACTF("Applying timeout settings from resumed session (%u ms).", - afl->fsrv.exec_tmout); - } else { ACTF("-t option specified. We'll use an exec timeout of %u ms.", @@ -2329,3 +2480,26 @@ void show_init_stats(afl_state_t *afl) { } +void update_calibration_time(afl_state_t *afl, u64 *time) { + + u64 cur = get_cur_time_us(); + afl->calibration_time_us += cur - *time; + *time = cur; + +} + +void update_trim_time(afl_state_t *afl, u64 *time) { + + u64 cur = get_cur_time_us(); + afl->trim_time_us += cur - *time; + *time = cur; + +} + +void update_sync_time(afl_state_t *afl, u64 *time) { + + u64 cur = get_cur_time_us(); + afl->sync_time_us += cur - *time; + *time = cur; + +} diff --git a/src/afl-fuzz-statsd.c b/src/afl-fuzz-statsd.c index e835c8ea..2e42ea9b 100644 --- a/src/afl-fuzz-statsd.c +++ b/src/afl-fuzz-statsd.c @@ -223,7 +223,7 @@ int statsd_format_metric(afl_state_t *afl, char *buff, size_t bufflen) { char tags[MAX_TAG_LEN * 2] = {0}; if (afl->statsd_tags_format) { - snprintf(tags, MAX_TAG_LEN * 2, afl->statsd_tags_format, afl->use_banner, + snprintf(tags, MAX_TAG_LEN * 2, afl->statsd_tags_format, afl->sync_id, VERSION); } diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 4339ddd2..70ab983c 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -5,11 +5,12 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and - Andrea Fioraldi <andreafioraldi@gmail.com> + Dominik Meier <mail@dmnk.co>, + Andrea Fioraldi <andreafioraldi@gmail.com>, and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -125,12 +126,20 @@ static void usage(u8 *argv0, int more_help) { "Required parameters:\n" " -i dir - input directory with test cases (or '-' to resume, " - "also see AFL_AUTORESUME)\n" + "also see \n" + " AFL_AUTORESUME)\n" " -o dir - output directory for fuzzer findings\n\n" "Execution control settings:\n" + " -P strategy - set fix mutation strategy: explore (focus on new " + "coverage),\n" + " exploit (focus on triggering crashes). You can also " + "set a\n" + " number of seconds after without any finds it switches " + "to\n" + " exploit mode, and back on new coverage (default: %u)\n" " -p schedule - power schedules compute a seed's performance score:\n" - " fast(default), explore, exploit, seek, rare, mmopt, " + " explore(default), fast, exploit, seek, rare, mmopt, " "coe, lin\n" " quad -- see docs/FAQ.md for more information\n" " -f file - location read by the fuzzed program (default: stdin " @@ -157,25 +166,29 @@ static void usage(u8 *argv0, int more_help) { "\n" "Mutator settings:\n" + " -a type - target input format, \"text\" or \"binary\" (default: " + "generic)\n" " -g minlength - set min length of generated fuzz input (default: 1)\n" " -G maxlength - set max length of generated fuzz input (default: " "%lu)\n" - " -D - enable deterministic fuzzing (once per queue entry)\n" " -L minutes - use MOpt(imize) mode and set the time limit for " "entering the\n" " pacemaker mode (minutes of no new finds). 0 = " "immediately,\n" " -1 = immediately and together with normal mutation.\n" + " Note: this option is usually not very effective\n" " -c program - enable CmpLog by specifying a binary compiled for " "it.\n" " if using QEMU/FRIDA or the fuzzing target is " "compiled\n" - " for CmpLog then just use -c 0.\n" + " for CmpLog then use '-c 0'. To disable Cmplog use '-c " + "-'.\n" " -l cmplog_opts - CmpLog configuration values (e.g. \"2ATR\"):\n" " 1=small files, 2=larger files (default), 3=all " "files,\n" " A=arithmetic solving, T=transformational solving,\n" - " R=random colorization bytes.\n\n" + " X=extreme transform solving, R=random colorization " + "bytes.\n\n" "Fuzzing behavior settings:\n" " -Z - sequential queue selection instead of weighted " "random\n" @@ -187,7 +200,8 @@ static void usage(u8 *argv0, int more_help) { "Test settings:\n" " -s seed - use a fixed seed for the RNG\n" - " -V seconds - fuzz for a specified time then terminate\n" + " -V seconds - fuzz for a specified time then terminate (fuzz time " + "only!)\n" " -E execs - fuzz for an approx. no. of total executions then " "terminate\n" " Note: not precise and can have several more " @@ -200,7 +214,8 @@ static void usage(u8 *argv0, int more_help) { " -F path - sync to a foreign fuzzer queue directory (requires " "-M, can\n" " be specified up to %u times)\n" - // " -d - skip deterministic fuzzing in -M mode\n" + " -z - skip the enhanced deterministic fuzzing\n" + " (note that the old -d and -D flags are ignored.)\n" " -T text - text banner to show on the screen\n" " -I command - execute this command/script when a new crash is " "found\n" @@ -212,7 +227,8 @@ static void usage(u8 *argv0, int more_help) { " -e ext - file extension for the fuzz test input file (if " "needed)\n" "\n", - argv0, EXEC_TIMEOUT, MEM_LIMIT, MAX_FILE, FOREIGN_SYNCS_MAX); + argv0, STRATEGY_SWITCH_TIME, EXEC_TIMEOUT, MEM_LIMIT, MAX_FILE, + FOREIGN_SYNCS_MAX); if (more_help > 1) { @@ -248,10 +264,12 @@ static void usage(u8 *argv0, int more_help) { "AFL_CYCLE_SCHEDULES: after completing a cycle, switch to a different -p schedule\n" "AFL_DEBUG: extra debugging output for Python mode trimming\n" "AFL_DEBUG_CHILD: do not suppress stdout/stderr from target\n" + "AFL_DISABLE_REDUNDANT: disable any queue item that is redundant\n" "AFL_DISABLE_TRIM: disable the trimming of test cases\n" "AFL_DUMB_FORKSRV: use fork server without feedback from target\n" "AFL_EXIT_WHEN_DONE: exit when all inputs are run and no new finds are found\n" "AFL_EXIT_ON_TIME: exit when no new coverage is found within the specified time\n" + "AFL_EXIT_ON_SEED_ISSUES: exit on any kind of seed issues\n" "AFL_EXPAND_HAVOC_NOW: immediately enable expand havoc mode (default: after 60\n" " minutes and a cycle without finds)\n" "AFL_FAST_CAL: limit the calibration stage to three cycles for speedup\n" @@ -262,11 +280,14 @@ static void usage(u8 *argv0, int more_help) { "AFL_IGNORE_PROBLEMS: do not abort fuzzing if an incorrect setup is detected\n" "AFL_IGNORE_PROBLEMS_COVERAGE: if set in addition to AFL_IGNORE_PROBLEMS - also\n" " ignore those libs for coverage\n" + "AFL_IGNORE_SEED_PROBLEMS: skip over crashes and timeouts in the seeds instead of\n" + " exiting\n" "AFL_IGNORE_TIMEOUTS: do not process or save any timeouts\n" "AFL_IGNORE_UNKNOWN_ENVS: don't warn on unknown env vars\n" "AFL_IMPORT_FIRST: sync and import test cases from other fuzzer instances first\n" "AFL_INPUT_LEN_MIN/AFL_INPUT_LEN_MAX: like -g/-G set min/max fuzz length produced\n" - "AFL_PIZZA_MODE: 1 - enforce pizza mode, 0 - disable for April 1st\n" + "AFL_PIZZA_MODE: 1 - enforce pizza mode, -1 - disable for April 1st,\n" + " 0 (default) - activate on April 1st\n" "AFL_KILL_SIGNAL: Signal ID delivered to child processes on timeout, etc.\n" " (default: SIGKILL)\n" "AFL_FORK_SERVER_KILL_SIGNAL: Kill signal for the fork server on termination\n" @@ -285,8 +306,14 @@ static void usage(u8 *argv0, int more_help) { "AFL_NO_FORKSRV: run target via execve instead of using the forkserver\n" "AFL_NO_SNAPSHOT: do not use the snapshot feature (if the snapshot lkm is loaded)\n" "AFL_NO_STARTUP_CALIBRATION: no initial seed calibration, start fuzzing at once\n" + "AFL_NO_WARN_INSTABILITY: no warn about instability issues on startup calibration\n" "AFL_NO_UI: switch status screen off\n" - + "AFL_NYX_AUX_SIZE: size of the Nyx auxiliary buffer. Must be a multiple of 4096.\n" + " Increase this value in case the crash reports are truncated.\n" + " Default value is 4096.\n" + "AFL_NYX_DISABLE_SNAPSHOT_MODE: disable snapshot mode (must be supported by the agent)\n" + "AFL_NYX_LOG: output NYX hprintf messages to another file\n" + "AFL_NYX_REUSE_SNAPSHOT: reuse an existing Nyx root snapshot\n" DYN_COLOR "AFL_PATH: path to AFL support binaries\n" @@ -295,8 +322,8 @@ static void usage(u8 *argv0, int more_help) { PERSISTENT_MSG - "AFL_POST_PROCESS_KEEP_ORIGINAL: save the file as it was prior post-processing to the queue,\n" - " but execute the post-processed one\n" + "AFL_POST_PROCESS_KEEP_ORIGINAL: save the file as it was prior post-processing to\n" + " the queue, but execute the post-processed one\n" "AFL_PRELOAD: LD_PRELOAD / DYLD_INSERT_LIBRARIES settings for target\n" "AFL_TARGET_ENV: pass extra environment variables to target\n" "AFL_SHUFFLE_QUEUE: reorder the input queue randomly on startup\n" @@ -307,18 +334,18 @@ static void usage(u8 *argv0, int more_help) { "AFL_STATSD_HOST: change default statsd host (default 127.0.0.1)\n" "AFL_STATSD_PORT: change default statsd port (default: 8125)\n" "AFL_STATSD_TAGS_FLAVOR: set statsd tags format (default: disable tags)\n" - " Supported formats are: 'dogstatsd', 'librato',\n" - " 'signalfx' and 'influxdb'\n" + " suported formats: dogstatsd, librato, signalfx, influxdb\n" "AFL_SYNC_TIME: sync time between fuzzing instances (in minutes)\n" + "AFL_FINAL_SYNC: sync a final time when exiting (will delay the exit!)\n" "AFL_NO_CRASH_README: do not create a README in the crashes directory\n" "AFL_TESTCACHE_SIZE: use a cache for testcases, improves performance (in MB)\n" "AFL_TMPDIR: directory to use for input file generation (ramdisk recommended)\n" "AFL_EARLY_FORKSERVER: force an early forkserver in an afl-clang-fast/\n" " afl-clang-lto/afl-gcc-fast target\n" - "AFL_PERSISTENT: enforce persistent mode (if __AFL_LOOP is in a shared lib\n" - "AFL_DEFER_FORKSRV: enforced deferred forkserver (__AFL_INIT is in a .so)\n" - "AFL_FUZZER_STATS_UPDATE_INTERVAL: interval to update fuzzer_stats file in seconds, " - "(default: 60, minimum: 1)\n" + "AFL_PERSISTENT: enforce persistent mode (if __AFL_LOOP is in a shared lib)\n" + "AFL_DEFER_FORKSRV: enforced deferred forkserver (__AFL_INIT is in a shared lib)\n" + "AFL_FUZZER_STATS_UPDATE_INTERVAL: interval to update fuzzer_stats file in\n" + " seconds (default: 60, minimum: 1)\n" "\n" ); @@ -357,6 +384,10 @@ static void usage(u8 *argv0, int more_help) { SAYF("Compiled with NO_SPLICING.\n"); #endif +#ifdef FANCY_BOXES_NO_UTF + SAYF("Compiled without UTF-8 support for line rendering in status screen.\n"); +#endif + #ifdef PROFILING SAYF("Compiled with PROFILING.\n"); #endif @@ -373,6 +404,12 @@ static void usage(u8 *argv0, int more_help) { SAYF("Compiled with _AFL_DOCUMENT_MUTATIONS.\n"); #endif +#ifdef _AFL_SPECIAL_PERFORMANCE + SAYF( + "Compiled with special performance options for this specific system, it " + "might not work on other platforms!\n"); +#endif + SAYF("For additional help please consult %s/README.md :)\n\n", doc_path); exit(1); @@ -458,6 +495,22 @@ int main(int argc, char **argv_orig, char **envp) { struct timeval tv; struct timezone tz; + doc_path = access(DOC_PATH, F_OK) != 0 ? (u8 *)"docs" : (u8 *)DOC_PATH; + + if (argc > 1 && strcmp(argv_orig[1], "--version") == 0) { + + printf("afl-fuzz" VERSION "\n"); + exit(0); + + } + + if (argc > 1 && strcmp(argv_orig[1], "--help") == 0) { + + usage(argv_orig[0], 1); + exit(0); + + } + #if defined USE_COLOR && defined ALWAYS_COLORED if (getenv("AFL_NO_COLOR") || getenv("AFL_NO_COLOUR")) { @@ -487,21 +540,72 @@ int main(int argc, char **argv_orig, char **envp) { SAYF(cCYA "afl-fuzz" VERSION cRST " based on afl by Michal Zalewski and a large online community\n"); - doc_path = access(DOC_PATH, F_OK) != 0 ? (u8 *)"docs" : (u8 *)DOC_PATH; - gettimeofday(&tv, &tz); rand_set_seed(afl, tv.tv_sec ^ tv.tv_usec ^ getpid()); afl->shmem_testcase_mode = 1; // we always try to perform shmem fuzzing - while ( - (opt = getopt( - argc, argv, - "+Ab:B:c:CdDe:E:hi:I:f:F:g:G:l:L:m:M:nNOo:p:RQs:S:t:T:UV:WXx:YZ")) > - 0) { + // still available: HjJkKqruvwz + while ((opt = getopt(argc, argv, + "+a:Ab:B:c:CdDe:E:f:F:g:G:hi:I:l:L:m:M:nNo:Op:P:QRs:S:t:" + "T:UV:WXx:YzZ")) > 0) { switch (opt) { + case 'a': + + if (!stricmp(optarg, "text") || !stricmp(optarg, "ascii") || + !stricmp(optarg, "txt") || !stricmp(optarg, "asc")) { + + afl->input_mode = 1; + + } else if (!stricmp(optarg, "bin") || !stricmp(optarg, "binary")) { + + afl->input_mode = 2; + + } else if (!stricmp(optarg, "def") || !stricmp(optarg, "default")) { + + afl->input_mode = 0; + + } else { + + FATAL("-a input mode needs to be \"text\" or \"binary\"."); + + } + + break; + + case 'P': + if (!stricmp(optarg, "explore") || !stricmp(optarg, "exploration")) { + + afl->fuzz_mode = 0; + afl->switch_fuzz_mode = 0; + + } else if (!stricmp(optarg, "exploit") || + + !stricmp(optarg, "exploitation")) { + + afl->fuzz_mode = 1; + afl->switch_fuzz_mode = 0; + + } else { + + if ((afl->switch_fuzz_mode = (u32)atoi(optarg)) > INT_MAX) { + + FATAL( + "Parameter for option -P must be \"explore\", \"exploit\" or a " + "number!"); + + } else { + + afl->switch_fuzz_mode *= 1000; + + } + + } + + break; + case 'g': afl->min_length = atoi(optarg); break; @@ -534,8 +638,23 @@ int main(int argc, char **argv_orig, char **envp) { case 'c': { - afl->shm.cmplog_mode = 1; - afl->cmplog_binary = ck_strdup(optarg); + if (strcmp(optarg, "-") == 0) { + + if (afl->shm.cmplog_mode) { + + ACTF("Disabling cmplog again because of '-c -'."); + afl->shm.cmplog_mode = 0; + afl->cmplog_binary = NULL; + + } + + } else { + + afl->shm.cmplog_mode = 1; + afl->cmplog_binary = ck_strdup(optarg); + + } + break; } @@ -845,12 +964,15 @@ int main(int argc, char **argv_orig, char **envp) { break; - case 'D': /* enforce deterministic */ + case 'd': + case 'D': /* old deterministic */ - afl->skip_deterministic = 0; + WARNF( + "Parameters -d and -D are deprecated, a new enhanced deterministic " + "fuzzing is active by default, to disable it use -z"); break; - case 'd': /* skip deterministic */ + case 'z': /* no deterministic */ afl->skip_deterministic = 1; break; @@ -1056,10 +1178,18 @@ int main(int argc, char **argv_orig, char **envp) { case 'A': afl->cmplog_enable_arith = 1; break; + case 's': + case 'S': + afl->cmplog_enable_scale = 1; + break; case 't': case 'T': afl->cmplog_enable_transform = 1; break; + case 'x': + case 'X': + afl->cmplog_enable_xtreme_transform = 1; + break; case 'r': case 'R': afl->cmplog_random_colorization = 1; @@ -1108,6 +1238,7 @@ int main(int argc, char **argv_orig, char **envp) { } + afl->old_seed_selection = 1; u64 limit_time_puppet2 = afl->limit_time_puppet * 60 * 1000; if ((s32)limit_time_puppet2 < afl->limit_time_puppet) { @@ -1221,6 +1352,10 @@ int main(int argc, char **argv_orig, char **envp) { } + WARNF( + "Note that the MOpt mode is not maintained and is not as effective " + "as normal havoc mode."); + } break; case 'h': @@ -1242,6 +1377,12 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->sync_id && strcmp(afl->sync_id, "addseeds") == 0) { + + FATAL("-M/-S name 'addseeds' is a reserved name, choose something else"); + + } + if (afl->is_main_node == 1 && afl->schedule != FAST && afl->schedule != EXPLORE) { @@ -1296,11 +1437,11 @@ int main(int argc, char **argv_orig, char **envp) { } #endif + + // silently disable deterministic mutation if custom mutators are used if (!afl->skip_deterministic && afl->afl_env.afl_custom_mutator_only) { - FATAL( - "Using -D determinstic fuzzing is incompatible with " - "AFL_CUSTOM_MUTATOR_ONLY!"); + afl->skip_deterministic = 1; } @@ -1396,9 +1537,9 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->sync_id) { - if (strlen(afl->sync_id) > 24) { + if (strlen(afl->sync_id) > 50) { - FATAL("sync_id max length is 24 characters"); + FATAL("sync_id max length is 50 characters"); } @@ -1424,7 +1565,11 @@ int main(int argc, char **argv_orig, char **envp) { setenv("__AFL_OUT_DIR", afl->out_dir, 1); - if (get_afl_env("AFL_DISABLE_TRIM")) { afl->disable_trim = 1; } + if (get_afl_env("AFL_DISABLE_TRIM") || get_afl_env("AFL_NO_TRIM")) { + + afl->disable_trim = 1; + + } if (getenv("AFL_NO_UI") && getenv("AFL_FORCE_UI")) { @@ -1436,8 +1581,7 @@ int main(int argc, char **argv_orig, char **envp) { if (!afl->use_banner) { afl->use_banner = argv[optind]; } - if (afl->shm.cmplog_mode && - (!strcmp("-", afl->cmplog_binary) || !strcmp("0", afl->cmplog_binary))) { + if (afl->shm.cmplog_mode && strcmp("0", afl->cmplog_binary) == 0) { afl->cmplog_binary = strdup(argv[optind]); @@ -1622,6 +1766,34 @@ int main(int argc, char **argv_orig, char **envp) { } + // Marker: ADD_TO_INJECTIONS + if (getenv("AFL_LLVM_INJECTIONS_ALL") || getenv("AFL_LLVM_INJECTIONS_SQL") || + getenv("AFL_LLVM_INJECTIONS_LDAP") || getenv("AFL_LLVM_INJECTIONS_XSS")) { + + OKF("Adding injection tokens to dictionary."); + if (getenv("AFL_LLVM_INJECTIONS_ALL") || + getenv("AFL_LLVM_INJECTIONS_SQL")) { + + add_extra(afl, "'\"\"'", 4); + + } + + if (getenv("AFL_LLVM_INJECTIONS_ALL") || + getenv("AFL_LLVM_INJECTIONS_LDAP")) { + + add_extra(afl, "*)(1=*))(|", 10); + + } + + if (getenv("AFL_LLVM_INJECTIONS_ALL") || + getenv("AFL_LLVM_INJECTIONS_XSS")) { + + add_extra(afl, "1\"><\"", 5); + + } + + } + OKF("Generating fuzz data with a length of min=%u max=%u", afl->min_length, afl->max_length); u32 min_alloc = MAX(64U, afl->min_length); @@ -1633,6 +1805,7 @@ int main(int argc, char **argv_orig, char **envp) { afl_realloc(AFL_BUF_PARAM(ex), min_alloc); afl->fsrv.use_fauxsrv = afl->non_instrumented_mode == 1 || afl->no_forkserver; + afl->fsrv.max_length = afl->max_length; #ifdef __linux__ if (!afl->fsrv.nyx_mode) { @@ -1657,6 +1830,10 @@ int main(int argc, char **argv_orig, char **envp) { check_cpu_governor(afl); #endif + #ifdef __APPLE__ + setenv("DYLD_NO_PIE", "1", 0); + #endif + if (getenv("LD_PRELOAD")) { WARNF( @@ -1746,16 +1923,6 @@ int main(int argc, char **argv_orig, char **envp) { check_if_tty(afl); if (afl->afl_env.afl_force_ui) { afl->not_on_tty = 0; } - if (afl->afl_env.afl_custom_mutator_only) { - - /* This ensures we don't proceed to havoc/splice */ - afl->custom_only = 1; - - /* Ensure we also skip all deterministic steps */ - afl->skip_deterministic = 1; - - } - get_core_count(afl); atexit(at_exit); @@ -1766,6 +1933,15 @@ int main(int argc, char **argv_orig, char **envp) { bind_to_free_cpu(afl); #endif /* HAVE_AFFINITY */ + #ifdef __linux__ + if (afl->fsrv.nyx_mode && afl->fsrv.nyx_bind_cpu_id == 0xFFFFFFFF) { + + afl->fsrv.nyx_bind_cpu_id = 0; + + } + + #endif + #ifdef __HAIKU__ /* Prioritizes performance over power saving */ set_scheduler_mode(SCHEDULER_MODE_LOW_LATENCY); @@ -1816,7 +1992,7 @@ int main(int argc, char **argv_orig, char **envp) { } - { + if (!getenv("AFL_CUSTOM_INFO_PROGRAM_ARGV")) { u8 envbuf[8096] = "", tmpbuf[8096] = ""; for (s32 i = optind + 1; i < argc; ++i) { @@ -1847,10 +2023,41 @@ int main(int argc, char **argv_orig, char **envp) { } - setenv("AFL_CUSTOM_INFO_OUT", afl->out_dir, 1); // same as __AFL_OUT_DIR + if (!getenv("AFL_CUSTOM_INFO_OUT")) { + + setenv("AFL_CUSTOM_INFO_OUT", afl->out_dir, 1); // same as __AFL_OUT_DIR + + } setup_custom_mutators(afl); + if (afl->afl_env.afl_custom_mutator_only) { + + if (!afl->custom_mutators_count) { + + if (afl->shm.cmplog_mode) { + + WARNF( + "No custom mutator loaded, using AFL_CUSTOM_MUTATOR_ONLY is " + "pointless and only allowed now to allow experiments with CMPLOG."); + + } else { + + FATAL( + "No custom mutator loaded but AFL_CUSTOM_MUTATOR_ONLY specified."); + + } + + } + + /* This ensures we don't proceed to havoc/splice */ + afl->custom_only = 1; + + /* Ensure we also skip all deterministic steps */ + afl->skip_deterministic = 1; + + } + if (afl->limit_time_sig > 0 && afl->custom_mutators_count) { if (afl->custom_only) { @@ -1874,6 +2081,17 @@ int main(int argc, char **argv_orig, char **envp) { } + /* Simply code if AFL_TMPDIR is used or not */ + if (!afl->afl_env.afl_tmpdir) { + + afl->tmp_dir = afl->out_dir; + + } else { + + afl->tmp_dir = afl->afl_env.afl_tmpdir; + + } + write_setup_file(afl, argc, argv); setup_cmdline_file(afl, argv + optind); @@ -1886,8 +2104,7 @@ int main(int argc, char **argv_orig, char **envp) { if (!afl->timeout_given) { find_timeout(afl); } // only for resumes! - if ((afl->tmp_dir = afl->afl_env.afl_tmpdir) != NULL && - !afl->in_place_resume) { + if (afl->afl_env.afl_tmpdir && !afl->in_place_resume) { char tmpfile[PATH_MAX]; @@ -1912,10 +2129,6 @@ int main(int argc, char **argv_orig, char **envp) { } - } else { - - afl->tmp_dir = afl->out_dir; - } /* If we don't have a file name chosen yet, use a safe default. */ @@ -1987,7 +2200,7 @@ int main(int argc, char **argv_orig, char **envp) { } - afl->fsrv.persistent_record_dir = alloc_printf("%s/crashes", afl->out_dir); + afl->fsrv.persistent_record_dir = alloc_printf("%s", afl->out_dir); } @@ -2253,7 +2466,7 @@ int main(int argc, char **argv_orig, char **envp) { } else { - ACTF("skipping initial seed calibration due option override"); + ACTF("skipping initial seed calibration due option override!"); usleep(1000); } @@ -2294,10 +2507,18 @@ int main(int argc, char **argv_orig, char **envp) { for (entry = 0; entry < afl->queued_items; ++entry) if (!afl->queue_buf[entry]->disabled) - if (afl->queue_buf[entry]->exec_us > max_ms) - max_ms = afl->queue_buf[entry]->exec_us; + if ((afl->queue_buf[entry]->exec_us / 1000) > max_ms) + max_ms = afl->queue_buf[entry]->exec_us / 1000; + + // Add 20% as a safety margin, capped to exec_tmout given in -t option + max_ms *= 1.2; + if (max_ms > afl->fsrv.exec_tmout) max_ms = afl->fsrv.exec_tmout; + + // Ensure that there is a sensible timeout even for very fast binaries + if (max_ms < 5) max_ms = 5; afl->fsrv.exec_tmout = max_ms; + afl->timeout_given = 1; } @@ -2330,8 +2551,6 @@ int main(int argc, char **argv_orig, char **envp) { } // (void)nice(-20); // does not improve the speed - // real start time, we reset, so this works correctly with -V - afl->start_time = get_cur_time(); #ifdef INTROSPECTION u32 prev_saved_crashes = 0, prev_saved_tmouts = 0; @@ -2352,6 +2571,9 @@ int main(int argc, char **argv_orig, char **envp) { OKF("Writing mutation introspection to '%s'", ifn); #endif + // real start time, we reset, so this works correctly with -V + afl->start_time = get_cur_time(); + while (likely(!afl->stop_soon)) { cull_queue(afl); @@ -2590,22 +2812,52 @@ int main(int argc, char **argv_orig, char **envp) { if (likely(!afl->old_seed_selection)) { - if (unlikely(prev_queued_items < afl->queued_items || - afl->reinit_table)) { + if (likely(afl->pending_favored && afl->smallest_favored >= 0)) { - // we have new queue entries since the last run, recreate alias table - prev_queued_items = afl->queued_items; - create_alias_table(afl); + afl->current_entry = afl->smallest_favored; - } + /* - do { + } else { - afl->current_entry = select_next_queue_entry(afl); + for (s32 iter = afl->queued_items - 1; iter >= 0; --iter) + { - } while (unlikely(afl->current_entry >= afl->queued_items)); + if (unlikely(afl->queue_buf[iter]->favored && + !afl->queue_buf[iter]->was_fuzzed)) { - afl->queue_cur = afl->queue_buf[afl->current_entry]; + afl->current_entry = iter; + break; + + } + + } + + */ + + afl->queue_cur = afl->queue_buf[afl->current_entry]; + + } else { + + if (unlikely(prev_queued_items < afl->queued_items || + afl->reinit_table)) { + + // we have new queue entries since the last run, recreate alias + // table + prev_queued_items = afl->queued_items; + create_alias_table(afl); + + } + + do { + + afl->current_entry = select_next_queue_entry(afl); + + } while (unlikely(afl->current_entry >= afl->queued_items)); + + afl->queue_cur = afl->queue_buf[afl->current_entry]; + + } } @@ -2667,13 +2919,34 @@ int main(int argc, char **argv_orig, char **envp) { } while (skipped_fuzz && afl->queue_cur && !afl->stop_soon); + u64 cur_time = get_cur_time(); + + if (likely(afl->switch_fuzz_mode && afl->fuzz_mode == 0 && + !afl->non_instrumented_mode) && + unlikely(cur_time > (likely(afl->last_find_time) ? afl->last_find_time + : afl->start_time) + + afl->switch_fuzz_mode)) { + + if (afl->afl_env.afl_no_ui) { + + ACTF( + "No new coverage found for %llu seconds, switching to exploitation " + "strategy.", + afl->switch_fuzz_mode / 1000); + + } + + afl->fuzz_mode = 1; + + } + if (likely(!afl->stop_soon && afl->sync_id)) { if (likely(afl->skip_deterministic)) { if (unlikely(afl->is_main_node)) { - if (unlikely(get_cur_time() > + if (unlikely(cur_time > (afl->sync_time >> 1) + afl->last_sync_time)) { if (!(sync_interval_cnt++ % (SYNC_INTERVAL / 3))) { @@ -2686,7 +2959,7 @@ int main(int argc, char **argv_orig, char **envp) { } else { - if (unlikely(get_cur_time() > afl->sync_time + afl->last_sync_time)) { + if (unlikely(cur_time > afl->sync_time + afl->last_sync_time)) { if (!(sync_interval_cnt++ % SYNC_INTERVAL)) { sync_fuzzers(afl); } @@ -2769,6 +3042,16 @@ stop_fuzzing: time_spent_working / afl->fsrv.total_execs); #endif + if (afl->afl_env.afl_final_sync) { + + SAYF(cYEL "[!] " cRST + "\nPerforming final sync, this make take some time ...\n"); + sync_fuzzers(afl); + write_bitmap(afl); + SAYF(cYEL "[!] " cRST "Done!\n\n"); + + } + if (afl->is_main_node) { u8 path[PATH_MAX]; @@ -2780,6 +3063,11 @@ stop_fuzzing: if (frida_afl_preload) { ck_free(frida_afl_preload); } fclose(afl->fsrv.plot_file); + + #ifdef INTROSPECTION + fclose(afl->fsrv.det_plot_file); + #endif + destroy_queue(afl); destroy_extras(afl); destroy_custom_mutators(afl); @@ -2795,7 +3083,7 @@ stop_fuzzing: afl_fsrv_deinit(&afl->fsrv); /* remove tmpfile */ - if (afl->tmp_dir != NULL && !afl->in_place_resume && afl->fsrv.out_file) { + if (!afl->in_place_resume && afl->fsrv.out_file) { (void)unlink(afl->fsrv.out_file); diff --git a/src/afl-gotcpu.c b/src/afl-gotcpu.c index 4f851099..6a3bd037 100644 --- a/src/afl-gotcpu.c +++ b/src/afl-gotcpu.c @@ -5,11 +5,11 @@ Originally written by Michal Zalewski Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-ld-lto.c b/src/afl-ld-lto.c index 420dd817..578552ba 100644 --- a/src/afl-ld-lto.c +++ b/src/afl-ld-lto.c @@ -5,11 +5,11 @@ Written by Marc Heuse <mh@mh-sec.de> for AFL++ Maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> Andrea Fioraldi <andreafioraldi@gmail.com> Dominik Maier <domenukk@gmail.com> - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,7 +23,9 @@ */ #define AFL_MAIN -#define _GNU_SOURCE +#ifndef _GNU_SOURCE + #define _GNU_SOURCE +#endif #include "config.h" #include "types.h" @@ -37,6 +39,7 @@ #include <time.h> #include <ctype.h> #include <fcntl.h> +#include <limits.h> #include <sys/stat.h> #include <sys/types.h> @@ -45,11 +48,6 @@ #include <dirent.h> -#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || \ - defined(__DragonFly__) - #include <limits.h> -#endif - #ifdef __APPLE__ #include <sys/syslimits.h> #endif @@ -280,7 +278,7 @@ int main(int argc, char **argv) { if (getenv("AFL_LD_PASSTHROUGH") != NULL) passthrough = 1; if (getenv("AFL_REAL_LD") != NULL) real_ld = getenv("AFL_REAL_LD"); - if (!afl_path || !*afl_path) afl_path = "/usr/local/lib/afl"; + if (!afl_path || !*afl_path) afl_path = AFL_PATH; setenv("AFL_LD_CALLER", "1", 1); diff --git a/src/afl-performance.c b/src/afl-performance.c index 04507410..6c6e3c8b 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -1,31 +1,18 @@ -/* - Written in 2019 by David Blackman and Sebastiano Vigna (vigna@acm.org) - - To the extent possible under law, the author has dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - See <https://creativecommons.org/publicdomain/zero/1.0/>. - - This is xoshiro256++ 1.0, one of our all-purpose, rock-solid generators. - It has excellent (sub-ns) speed, a state (256 bits) that is large - enough for any parallel application, and it passes all tests we are - aware of. - - For generating just floating-point numbers, xoshiro256+ is even faster. - - The state must be seeded so that it is not everywhere zero. If you have - a 64-bit seed, we suggest to seed a splitmix64 generator and use its - output to fill s[]. -*/ - #include <stdint.h> #include "afl-fuzz.h" #include "types.h" -#define XXH_INLINE_ALL -#include "xxhash.h" -#undef XXH_INLINE_ALL +#ifdef _HAVE_AVX2 + #define T1HA0_AESNI_AVAILABLE 1 + #define T1HA_USE_FAST_ONESHOT_READ 1 + #define T1HA_USE_INDIRECT_FUNCTIONS 1 + #define T1HA_IA32AES_NAME XXH3_64bits + #include "t1ha0_ia32aes_b.h" +#else + #define XXH_INLINE_ALL + #include "xxhash.h" + #undef XXH_INLINE_ALL +#endif void rand_set_seed(afl_state_t *afl, s64 init_seed) { @@ -108,3 +95,313 @@ inline u64 hash64(u8 *key, u32 len, u64 seed) { } +// Public domain SHA1 implementation copied from: +// https://github.com/x42/liboauth/blob/7001b8256cd654952ec2515b055d2c5b243be600/src/sha1.c + +/* This code is public-domain - it is based on libcrypt + * placed in the public domain by Wei Dai and other contributors. + */ +// gcc -Wall -DSHA1TEST -o sha1test sha1.c && ./sha1test + +#include <stdint.h> +#include <string.h> + +#ifdef __BIG_ENDIAN__ + #define SHA_BIG_ENDIAN +#elif defined __LITTLE_ENDIAN__ +/* override */ +#elif defined __BYTE_ORDER + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #define SHA_BIG_ENDIAN + #endif +#else // ! defined __LITTLE_ENDIAN__ + #include <endian.h> // machine/endian.h + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #define SHA_BIG_ENDIAN + #endif +#endif + +/* header */ + +#define HASH_LENGTH 20 +#define BLOCK_LENGTH 64 + +typedef struct sha1nfo { + + uint32_t buffer[BLOCK_LENGTH / 4]; + uint32_t state[HASH_LENGTH / 4]; + uint32_t byteCount; + uint8_t bufferOffset; + uint8_t keyBuffer[BLOCK_LENGTH]; + uint8_t innerHash[HASH_LENGTH]; + +} sha1nfo; + +/* public API - prototypes - TODO: doxygen*/ + +/** + */ +void sha1_init(sha1nfo *s); +/** + */ +void sha1_writebyte(sha1nfo *s, uint8_t data); +/** + */ +void sha1_write(sha1nfo *s, const char *data, size_t len); +/** + */ +uint8_t *sha1_result(sha1nfo *s); +/** + */ +void sha1_initHmac(sha1nfo *s, const uint8_t *key, int keyLength); +/** + */ +uint8_t *sha1_resultHmac(sha1nfo *s); + +/* code */ +#define SHA1_K0 0x5a827999 +#define SHA1_K20 0x6ed9eba1 +#define SHA1_K40 0x8f1bbcdc +#define SHA1_K60 0xca62c1d6 + +void sha1_init(sha1nfo *s) { + + s->state[0] = 0x67452301; + s->state[1] = 0xefcdab89; + s->state[2] = 0x98badcfe; + s->state[3] = 0x10325476; + s->state[4] = 0xc3d2e1f0; + s->byteCount = 0; + s->bufferOffset = 0; + +} + +uint32_t sha1_rol32(uint32_t number, uint8_t bits) { + + return ((number << bits) | (number >> (32 - bits))); + +} + +void sha1_hashBlock(sha1nfo *s) { + + uint8_t i; + uint32_t a, b, c, d, e, t; + + a = s->state[0]; + b = s->state[1]; + c = s->state[2]; + d = s->state[3]; + e = s->state[4]; + for (i = 0; i < 80; i++) { + + if (i >= 16) { + + t = s->buffer[(i + 13) & 15] ^ s->buffer[(i + 8) & 15] ^ + s->buffer[(i + 2) & 15] ^ s->buffer[i & 15]; + s->buffer[i & 15] = sha1_rol32(t, 1); + + } + + if (i < 20) { + + t = (d ^ (b & (c ^ d))) + SHA1_K0; + + } else if (i < 40) { + + t = (b ^ c ^ d) + SHA1_K20; + + } else if (i < 60) { + + t = ((b & c) | (d & (b | c))) + SHA1_K40; + + } else { + + t = (b ^ c ^ d) + SHA1_K60; + + } + + t += sha1_rol32(a, 5) + e + s->buffer[i & 15]; + e = d; + d = c; + c = sha1_rol32(b, 30); + b = a; + a = t; + + } + + s->state[0] += a; + s->state[1] += b; + s->state[2] += c; + s->state[3] += d; + s->state[4] += e; + +} + +void sha1_addUncounted(sha1nfo *s, uint8_t data) { + + uint8_t *const b = (uint8_t *)s->buffer; +#ifdef SHA_BIG_ENDIAN + b[s->bufferOffset] = data; +#else + b[s->bufferOffset ^ 3] = data; +#endif + s->bufferOffset++; + if (s->bufferOffset == BLOCK_LENGTH) { + + sha1_hashBlock(s); + s->bufferOffset = 0; + + } + +} + +void sha1_writebyte(sha1nfo *s, uint8_t data) { + + ++s->byteCount; + sha1_addUncounted(s, data); + +} + +void sha1_write(sha1nfo *s, const char *data, size_t len) { + + for (; len--;) + sha1_writebyte(s, (uint8_t)*data++); + +} + +void sha1_pad(sha1nfo *s) { + + // Implement SHA-1 padding (fips180-2 Ā§5.1.1) + + // Pad with 0x80 followed by 0x00 until the end of the block + sha1_addUncounted(s, 0x80); + while (s->bufferOffset != 56) + sha1_addUncounted(s, 0x00); + + // Append length in the last 8 bytes + sha1_addUncounted(s, 0); // We're only using 32 bit lengths + sha1_addUncounted(s, 0); // But SHA-1 supports 64 bit lengths + sha1_addUncounted(s, 0); // So zero pad the top bits + sha1_addUncounted(s, s->byteCount >> 29); // Shifting to multiply by 8 + sha1_addUncounted( + s, s->byteCount >> 21); // as SHA-1 supports bitstreams as well as + sha1_addUncounted(s, s->byteCount >> 13); // byte. + sha1_addUncounted(s, s->byteCount >> 5); + sha1_addUncounted(s, s->byteCount << 3); + +} + +uint8_t *sha1_result(sha1nfo *s) { + + // Pad to complete the last block + sha1_pad(s); + +#ifndef SHA_BIG_ENDIAN + // Swap byte order back + int i; + for (i = 0; i < 5; i++) { + + s->state[i] = (((s->state[i]) << 24) & 0xff000000) | + (((s->state[i]) << 8) & 0x00ff0000) | + (((s->state[i]) >> 8) & 0x0000ff00) | + (((s->state[i]) >> 24) & 0x000000ff); + + } + +#endif + + // Return pointer to hash (20 characters) + return (uint8_t *)s->state; + +} + +#define HMAC_IPAD 0x36 +#define HMAC_OPAD 0x5c + +void sha1_initHmac(sha1nfo *s, const uint8_t *key, int keyLength) { + + uint8_t i; + memset(s->keyBuffer, 0, BLOCK_LENGTH); + if (keyLength > BLOCK_LENGTH) { + + // Hash long keys + sha1_init(s); + for (; keyLength--;) + sha1_writebyte(s, *key++); + memcpy(s->keyBuffer, sha1_result(s), HASH_LENGTH); + + } else { + + // Block length keys are used as is + memcpy(s->keyBuffer, key, keyLength); + + } + + // Start inner hash + sha1_init(s); + for (i = 0; i < BLOCK_LENGTH; i++) { + + sha1_writebyte(s, s->keyBuffer[i] ^ HMAC_IPAD); + + } + +} + +uint8_t *sha1_resultHmac(sha1nfo *s) { + + uint8_t i; + // Complete inner hash + memcpy(s->innerHash, sha1_result(s), HASH_LENGTH); + // Calculate outer hash + sha1_init(s); + for (i = 0; i < BLOCK_LENGTH; i++) + sha1_writebyte(s, s->keyBuffer[i] ^ HMAC_OPAD); + for (i = 0; i < HASH_LENGTH; i++) + sha1_writebyte(s, s->innerHash[i]); + return sha1_result(s); + +} + +// End public domain SHA1 implementation + +void sha1(const u8 *data, size_t len, u8 *out) { + + sha1nfo s; + sha1_init(&s); + sha1_write(&s, (const char *)data, len); + memcpy(out, sha1_result(&s), HASH_LENGTH); + +} + +char *sha1_hex(const u8 *data, size_t len) { + + u8 digest[HASH_LENGTH]; + sha1(data, len, digest); + u8 *hex = ck_alloc(HASH_LENGTH * 2 + 1); + for (size_t i = 0; i < HASH_LENGTH; ++i) { + + sprintf((char *)(hex + i * 2), "%02x", digest[i]); + + } + + return hex; + +} + +char *sha1_hex_for_file(const char *fname, u32 len) { + + int fd = open(fname, O_RDONLY); + if (fd < 0) { PFATAL("Unable to open '%s'", fname); } + + u32 read_len = MIN(len, (u32)MAX_FILE); + u8 *tmp = ck_alloc(read_len); + ck_read(fd, tmp, read_len, fname); + + close(fd); + + char *hex = sha1_hex(tmp, read_len); + ck_free(tmp); + return hex; + +} + diff --git a/src/afl-sharedmem.c b/src/afl-sharedmem.c index a2c81586..8f685633 100644 --- a/src/afl-sharedmem.c +++ b/src/afl-sharedmem.c @@ -7,11 +7,11 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 9c029035..7e875040 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -7,12 +7,12 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> and Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -111,8 +111,9 @@ static sharedmem_t *shm_fuzz; static const u8 count_class_human[256] = { - [0] = 0, [1] = 1, [2] = 2, [3] = 3, [4] = 4, - [8] = 5, [16] = 6, [32] = 7, [128] = 8 + [0] = 0, [1] = 1, [2] = 2, [3] = 3, + [4 ... 7] = 4, [8 ... 15] = 5, [16 ... 31] = 6, [32 ... 127] = 7, + [128 ... 255] = 8 }; @@ -177,7 +178,8 @@ fsrv_run_result_t fuzz_run_target(afl_state_t *afl, afl_forkserver_t *fsrv, void classify_counts(afl_forkserver_t *fsrv) { u8 *mem = fsrv->trace_bits; - const u8 *map = binary_mode ? count_class_binary : count_class_human; + const u8 *map = (binary_mode || collect_coverage) ? count_class_binary + : count_class_human; u32 i = map_size; @@ -239,13 +241,7 @@ static void analyze_results(afl_forkserver_t *fsrv) { u32 i; for (i = 0; i < map_size; i++) { - if (fsrv->trace_bits[i]) { - - total += fsrv->trace_bits[i]; - if (fsrv->trace_bits[i] > highest) highest = fsrv->trace_bits[i]; - if (!coverage_map[i]) { coverage_map[i] = 1; } - - } + if (fsrv->trace_bits[i]) { coverage_map[i] |= fsrv->trace_bits[i]; } } @@ -328,7 +324,7 @@ static u32 write_results_to_file(afl_forkserver_t *fsrv, u8 *outfile) { if (cmin_mode) { - fprintf(f, "%u%u\n", fsrv->trace_bits[i], i); + fprintf(f, "%u%03u\n", i, fsrv->trace_bits[i]); } else { @@ -423,9 +419,9 @@ static void showmap_run_target_forkserver(afl_forkserver_t *fsrv, u8 *mem, } - if (fsrv->trace_bits[0] == 1) { + if (fsrv->trace_bits[0]) { - fsrv->trace_bits[0] = 0; + fsrv->trace_bits[0] -= 1; have_coverage = true; } else { @@ -654,9 +650,9 @@ static void showmap_run_target(afl_forkserver_t *fsrv, char **argv) { } - if (fsrv->trace_bits[0] == 1) { + if (fsrv->trace_bits[0]) { - fsrv->trace_bits[0] = 0; + fsrv->trace_bits[0] -= 1; have_coverage = true; } else { @@ -1337,6 +1333,8 @@ int main(int argc, char **argv_orig, char **envp) { } + if (collect_coverage) { binary_mode = false; } // ensure this + if (optind == argc || !out_file) { usage(argv[0]); } if (in_dir && in_filelist) { FATAL("you can only specify either -i or -I"); } @@ -1609,6 +1607,7 @@ int main(int argc, char **argv_orig, char **envp) { if (in_dir || in_filelist) { afl->fsrv.dev_urandom_fd = open("/dev/urandom", O_RDONLY); + if (afl->fsrv.dev_urandom_fd < 0) { PFATAL("Unable to open /dev/urandom"); } afl->afl_env.afl_custom_mutator_library = getenv("AFL_CUSTOM_MUTATOR_LIBRARY"); afl->afl_env.afl_python_module = getenv("AFL_PYTHON_MODULE"); @@ -1674,7 +1673,6 @@ int main(int argc, char **argv_orig, char **envp) { if ((coverage_map = (u8 *)malloc(map_size + 64)) == NULL) FATAL("coult not grab memory"); edges_only = false; - raw_instr_output = true; } diff --git a/src/afl-tmin.c b/src/afl-tmin.c index e7442d1d..23e0ff13 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -7,12 +7,12 @@ Forkserver design by Jann Horn <jannhorn@googlemail.com> Now maintained by Marc Heuse <mh@mh-sec.de>, - Heiko EiĆfeldt <heiko.eissfeldt@hexco.de> and + Heiko Eissfeldt <heiko.eissfeldt@hexco.de> and Andrea Fioraldi <andreafioraldi@gmail.com> and Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -82,6 +82,8 @@ static u8 crash_mode, /* Crash-centric mode? */ remove_shm = 1, /* remove shmem on exit? */ debug; /* debug mode */ +static u32 del_len_limit = 1; /* Minimum block deletion length */ + static volatile u8 stop_soon; /* Ctrl-C pressed? */ static afl_forkserver_t *fsrv; @@ -480,7 +482,7 @@ next_del_blksize: } - if (del_len > 1 && in_len >= 1) { + if (del_len > del_len_limit && in_len >= 1) { del_len /= 2; goto next_del_blksize; @@ -796,8 +798,9 @@ static void usage(u8 *argv0) { "Minimization settings:\n" " -e - solve for edge coverage only, ignore hit counts\n" - " -x - treat non-zero exit codes as crashes\n\n" - " -H - minimize a hang (hang mode)\n" + " -l bytes - set minimum block deletion length to speed up minimization\n" + " -x - treat non-zero exit codes as crashes\n" + " -H - minimize a hang (hang mode)\n\n" "For additional tips, please consult %s/README.md.\n\n" @@ -829,8 +832,9 @@ static void usage(u8 *argv0) { int main(int argc, char **argv_orig, char **envp) { - s32 opt; - u8 mem_limit_given = 0, timeout_given = 0, unicorn_mode = 0, use_wine = 0; + s32 opt; + u8 mem_limit_given = 0, timeout_given = 0, unicorn_mode = 0, use_wine = 0, + del_limit_given = 0; char **use_argv; char **argv = argv_cpy_dup(argc, argv_orig); @@ -846,7 +850,7 @@ int main(int argc, char **argv_orig, char **envp) { SAYF(cCYA "afl-tmin" VERSION cRST " by Michal Zalewski\n"); - while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeAOQUWXYHh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:l:B:xeAOQUWXYHh")) > 0) { switch (opt) { @@ -1055,6 +1059,24 @@ int main(int argc, char **argv_orig, char **envp) { read_bitmap(optarg, mask_bitmap, map_size); break; + case 'l': + if (del_limit_given) { FATAL("Multiple -l options not supported"); } + del_limit_given = 1; + + if (!optarg) { FATAL("Wrong usage of -l"); } + + if (optarg[0] == '-') { FATAL("Dangerously low value of -l"); } + + del_len_limit = atoi(optarg); + + if (del_len_limit < 1 || del_len_limit > TMIN_MAX_FILE) { + + FATAL("Value of -l out of range between 1 and TMIN_MAX_FILE"); + + } + + break; + case 'h': usage(argv[0]); return -1; diff --git a/src/hashmap.c b/src/hashmap.c new file mode 100644 index 00000000..a0a9283c --- /dev/null +++ b/src/hashmap.c @@ -0,0 +1,149 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdbool.h> +#include "types.h" +#define TABLE_SIZE 10007 // Use a prime number for better distribution + +typedef struct HashNode { + + uint64_t key; + struct HashNode *next; + +} HashNode; + +typedef struct HashMap { + + HashNode **table; + +} HashMap; + +static HashMap *_hashmap; + +void hashmap_reset() { + + if (unlikely(!_hashmap)) { + + _hashmap = (HashMap *)malloc(sizeof(HashMap)); + _hashmap->table = (HashNode **)malloc(sizeof(HashNode *) * TABLE_SIZE); + memset((char *)_hashmap->table, 0, sizeof(HashNode *) * TABLE_SIZE); + + } else { + + for (int i = 0; i < TABLE_SIZE; i++) { + + HashNode *node = _hashmap->table[i]; + while (node) { + + HashNode *temp = node; + node = node->next; + free(temp); + + } + + } + + memset((char *)_hashmap->table, 0, sizeof(HashNode *) * TABLE_SIZE); + + } + +} + +static inline unsigned int hash(uint64_t key) { + + return key % TABLE_SIZE; + +} + +// type must be below 8 +bool hashmap_search_and_add(uint8_t type, uint64_t key) { + + if (unlikely(type >= 8)) return false; + uint64_t val = (key & 0xf8ffffffffffffff) + (type << 56); + unsigned int index = hash(val); + HashNode *node = _hashmap->table[index]; + while (node) { + + if (node->key == val) return true; + node = node->next; + + } + + // not found so add it + node = (HashNode *)malloc(sizeof(HashNode)); + node->key = val; + node->next = _hashmap->table[index]; + _hashmap->table[index] = node; + + return false; + +} + +// type must be below 8 +bool hashmap_search_and_add_ptr(uint8_t type, u8 *key) { + + if (unlikely(type >= 8)) return false; + uint64_t key_t = 0; + memcpy(((char *)key_t) + (7 - type), key, type + 1); + return hashmap_search_and_add(type, key_t); + +} + +/* below is not used */ + +void hashmap_insert(uint64_t key) { + + unsigned int index = hash(key); + HashNode *node = (HashNode *)malloc(sizeof(HashNode)); + node->key = key; + node->next = _hashmap->table[index]; + _hashmap->table[index] = node; + +} + +bool hashmap_search(uint64_t key) { + + unsigned int index = hash(key); + HashNode *node = _hashmap->table[index]; + while (node) { + + if (node->key == key) return true; + node = node->next; + + } + + return false; + +} + +void delete(uint64_t key) { + + unsigned int index = hash(key); + HashNode *prev = NULL, *node = _hashmap->table[index]; + while (node) { + + if (node->key == key) { + + if (prev) + prev->next = node->next; + else + _hashmap->table[index] = node->next; + free(node); + return; + + } + + prev = node; + node = node->next; + + } + +} + +void freeHashMap(HashMap *map) { + + free(_hashmap->table); + free(map); + +} + diff --git a/test-instr.c b/test-instr.c index 1d9f2e6e..28552893 100644 --- a/test-instr.c +++ b/test-instr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: @@ -24,7 +24,7 @@ int main(int argc, char **argv) { - int fd = 0; + int fd = 0, cnt; char buff[8]; char *buf = buff; @@ -32,7 +32,6 @@ int main(int argc, char **argv) { if (argc == 2) { buf = argv[1]; - printf("Input %s - ", buf); } else { @@ -47,15 +46,19 @@ int main(int argc, char **argv) { } - if (read(fd, buf, sizeof(buf)) < 1) { + if ((cnt = read(fd, buf, sizeof(buf) - 1)) < 1) { printf("Hum?\n"); return 1; } + buf[cnt] = 0; + } + if (getenv("AFL_DEBUG")) fprintf(stderr, "test-instr: %s\n", buf); + // we support three input cases (plus a 4th if stdin is used but there is no // input) switch (buf[0]) { diff --git a/test/test-all.sh b/test/test-all.sh index 3cb692ca..65cfb812 100755 --- a/test/test-all.sh +++ b/test/test-all.sh @@ -16,6 +16,8 @@ . ./test-frida-mode.sh +. ./test-nyx-mode.sh + . ./test-unicorn-mode.sh . ./test-custom-mutators.sh diff --git a/test/test-basic.sh b/test/test-basic.sh index 5bb2ca28..7005d3ce 100755 --- a/test/test-basic.sh +++ b/test/test-basic.sh @@ -2,6 +2,7 @@ . ./test-pre.sh +OS=$(uname -s) AFL_GCC=afl-gcc $ECHO "$BLUE[*] Testing: ${AFL_GCC}, afl-showmap, afl-fuzz, afl-cmin and afl-tmin" @@ -28,7 +29,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc rm -f test-instr.plain.0 test-instr.plain.1 SKIP= TUPLES=`echo 1|AFL_QUIET=1 ../afl-showmap -m ${MEM_LIMIT} -o /dev/null -- ./test-instr.plain 2>&1 | grep Captur | awk '{print$3}'` - test "$TUPLES" -gt 1 -a "$TUPLES" -lt 12 && { + test "$TUPLES" -gt 1 -a "$TUPLES" -lt 22 && { $ECHO "$GREEN[+] ${AFL_GCC} run reported $TUPLES instrumented locations which is fine" } || { $ECHO "$RED[!] ${AFL_GCC} instrumentation produces weird numbers: $TUPLES" @@ -61,7 +62,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc } # now we want to be sure that afl-fuzz is working # make sure crash reporter is disabled on Mac OS X - (test "$(uname -s)" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { + (test "$OS" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { $ECHO "$RED[!] we cannot run afl-fuzz with enabled crash reporter. Run 'sudo sh afl-system-config'.$RESET" true }) || { @@ -84,16 +85,20 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc } echo 000000000000000000000000 > in/in2 echo 111 > in/in3 - mkdir -p in2 - ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? - CNT=`ls in2/* 2>/dev/null | wc -l` - case "$CNT" in - *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; - *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" - CODE=1 - ;; - esac - rm -f in2/in* + test "$OS" = "Darwin" && { + $ECHO "$GREY[*] afl-cmin not available on macOS, cannot test afl-cmin" + } || { + mkdir -p in2 + ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? + CNT=`ls in2/* 2>/dev/null | wc -l` + case "$CNT" in + *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" + CODE=1 + ;; + esac + rm -f in2/in* + } export AFL_QUIET=1 if command -v bash >/dev/null ; then { ../afl-cmin.bash -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null @@ -152,7 +157,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc } rm -f test-instr.plain.0 test-instr.plain.1 TUPLES=`echo 1|AFL_QUIET=1 ../afl-showmap -m ${MEM_LIMIT} -o /dev/null -- ./test-instr.plain 2>&1 | grep Captur | awk '{print$3}'` - test "$TUPLES" -gt 1 -a "$TUPLES" -lt 12 && { + test "$TUPLES" -gt 1 -a "$TUPLES" -lt 22 && { $ECHO "$GREEN[+] ${AFL_CLANG} run reported $TUPLES instrumented locations which is fine" } || { $ECHO "$RED[!] ${AFL_CLANG} instrumentation produces weird numbers: $TUPLES" @@ -182,7 +187,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc } # now we want to be sure that afl-fuzz is working # make sure crash reporter is disabled on Mac OS X - (test "$(uname -s)" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { + (test "$OS" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { $ECHO "$RED[!] we cannot run afl-fuzz with enabled crash reporter. Run 'sudo sh afl-system-config'.$RESET" true }) || { @@ -204,25 +209,29 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc } } echo 000000000000000000000000 > in/in2 - echo AAA > in/in3 - mkdir -p in2 - ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? - CNT=`ls in2/* 2>/dev/null | wc -l` - case "$CNT" in - *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; - \ *1|1) { # allow leading whitecase for portability - test -s in2/* && $ECHO "$YELLOW[?] afl-cmin did minimize to one testcase. This can be a bug or due compiler optimization." - test -s in2/* || { - $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" - CODE=1 + echo AAA > in/in2 + test "$OS" = "Darwin" && { + $ECHO "$GREY[*] afl-cmin not available on macOS, cannot test afl-cmin" + } || { + mkdir -p in2 + ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? + CNT=`ls in2/* 2>/dev/null | wc -l` + case "$CNT" in + *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + \ *1|1) { # allow leading whitecase for portability + test -s in2/* && $ECHO "$YELLOW[?] afl-cmin did minimize to one testcase. This can be a bug or due compiler optimization." + test -s in2/* || { + $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" + CODE=1 + } } - } - ;; - *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" - CODE=1 - ;; - esac - rm -f in2/in* + ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" + CODE=1 + ;; + esac + rm -f in2/in* + } export AFL_QUIET=1 if command -v bash >/dev/null ; then { ../afl-cmin.bash -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null diff --git a/test/test-cmplog.c b/test/test-cmplog.c index bd1b73e3..0c91b21c 100644 --- a/test/test-cmplog.c +++ b/test/test-cmplog.c @@ -8,16 +8,11 @@ int LLVMFuzzerTestOneInput(const uint8_t *buf, size_t i) { - if (i < 30) return -1; + if (i < 15) return -1; if (buf[0] != 'A') return 0; - if (buf[1] != 'B') return 0; - if (buf[2] != 'C') return 0; - if (buf[3] != 'D') return 0; - int *icmp = (int *)(buf + 4); + int *icmp = (int *)(buf + 1); if (*icmp != 0x69694141) return 0; - if (memcmp(buf + 8, "1234", 4) || memcmp(buf + 12, "EFGH", 4)) return 0; - if (strncmp(buf + 16, "IJKL", 4) == 0 && strcmp(buf + 20, "DEADBEEF") == 0) - abort(); + if (memcmp(buf + 5, "1234EF", 6) == 0) abort(); return 0; } diff --git a/test/test-compilers.sh b/test/test-compilers.sh new file mode 100755 index 00000000..b47cf38d --- /dev/null +++ b/test/test-compilers.sh @@ -0,0 +1,7 @@ +#!/bin/sh +echo Testing compilers ... +for cc in afl-cc afl-gcc afl-clang afl-clang-fast afl-clang-lto afl-gcc-fast; do + test -e ../$cc && { { ../$cc -o t ../test-instr.c >/dev/null 2<&1 && echo Success: $cc ; } || echo Failing: $cc ; } || echo Missing: $cc +done +rm -f t +echo Done! diff --git a/test/test-custom-mutators.sh b/test/test-custom-mutators.sh index 49feedc0..8c8b0ad3 100755 --- a/test/test-custom-mutators.sh +++ b/test/test-custom-mutators.sh @@ -38,7 +38,7 @@ test -e test-custom-mutator.c -a -e ${CUSTOM_MUTATOR_PATH}/example.c -a -e ${CUS # Run afl-fuzz w/ the C mutator $ECHO "$GREY[*] running afl-fuzz for the C mutator, this will take approx 10 seconds" { - AFL_CUSTOM_MUTATOR_LIBRARY=./libexamplemutator.so AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V07 -m ${MEM_LIMIT} -i in -o out -- ./test-custom-mutator >>errors 2>&1 + AFL_CUSTOM_MUTATOR_LIBRARY=./libexamplemutator.so AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V07 -m ${MEM_LIMIT} -i in -o out -d -- ./test-custom-mutator >>errors 2>&1 } >>errors 2>&1 # Check results @@ -58,7 +58,7 @@ test -e test-custom-mutator.c -a -e ${CUSTOM_MUTATOR_PATH}/example.c -a -e ${CUS # Run afl-fuzz w/ multiple C mutators $ECHO "$GREY[*] running afl-fuzz with multiple custom C mutators, this will take approx 10 seconds" { - AFL_CUSTOM_MUTATOR_LIBRARY="./libexamplemutator.so;./libexamplemutator2.so" AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V07 -m ${MEM_LIMIT} -i in -o out -- ./test-multiple-mutators >>errors 2>&1 + AFL_CUSTOM_MUTATOR_LIBRARY="./libexamplemutator.so;./libexamplemutator2.so" AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V07 -m ${MEM_LIMIT} -i in -o out -d -- ./test-multiple-mutators >>errors 2>&1 } >>errors 2>&1 test -n "$( ls out/default/crashes/id:000000* 2>/dev/null )" && { # TODO: update here diff --git a/test/test-frida-mode.sh b/test/test-frida-mode.sh index 3ae84656..8c528da5 100755 --- a/test/test-frida-mode.sh +++ b/test/test-frida-mode.sh @@ -39,7 +39,7 @@ test -e ../afl-frida-trace.so && { test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc" -o "$SYS" = "aarch64" -o ! "${SYS%%arm*}" && { $ECHO "$GREY[*] running afl-fuzz for frida_mode cmplog, this will take approx 10 seconds" { - ../afl-fuzz -m none -V07 -O -c 0 -i in -o out -- ./test-compcov >>errors 2>&1 + ../afl-fuzz -m none -V07 -O -c 0 -l 3 -i in -o out -- ./test-compcov >>errors 2>&1 } >>errors 2>&1 test -n "$( ls out/default/queue/id:000003* 2>/dev/null )" && { $ECHO "$GREEN[+] afl-fuzz is working correctly with frida_mode cmplog" diff --git a/test/test-libextensions.sh b/test/test-libextensions.sh index 40a898c8..f7f86de5 100755 --- a/test/test-libextensions.sh +++ b/test/test-libextensions.sh @@ -5,7 +5,7 @@ test -z "$AFL_CC" && unset AFL_CC $ECHO "$BLUE[*] Testing: shared library extensions" -cc $CFLAGS -o test-compcov test-compcov.c > /dev/null 2>&1 +cc $CFLAGS -O0 -o test-compcov test-compcov.c > /dev/null 2>&1 test -e ../libtokencap.so && { AFL_TOKEN_FILE=token.out LD_PRELOAD=../libtokencap.so DYLD_INSERT_LIBRARIES=../libtokencap.so DYLD_FORCE_FLAT_NAMESPACE=1 ./test-compcov foobar > /dev/null 2>&1 grep -q BUGMENOT token.out > /dev/null 2>&1 && { diff --git a/test/test-llvm.sh b/test/test-llvm.sh index 714bda93..4dd35e6e 100755 --- a/test/test-llvm.sh +++ b/test/test-llvm.sh @@ -2,6 +2,8 @@ . ./test-pre.sh +OS=$(uname -s) + $ECHO "$BLUE[*] Testing: llvm_mode, afl-showmap, afl-fuzz, afl-cmin and afl-tmin" test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 @@ -60,7 +62,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { $ECHO "$RED[!] llvm_mode threadsafe instrumentation failed" CODE=1 } - rm -f test-instr.ts.0 test-instr.ts.1 + rm -f test-instr.ts.0 test-instr.ts.1 test-instr.ts } || { $ECHO "$RED[!] llvm_mode (threadsafe) failed" CODE=1 @@ -123,7 +125,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { } # now we want to be sure that afl-fuzz is working # make sure crash reporter is disabled on Mac OS X - (test "$(uname -s)" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { + (test "$OS" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { $ECHO "$RED[!] we cannot run afl-fuzz with enabled crash reporter. Run 'sudo sh afl-system-config'.$RESET" CODE=1 true @@ -146,18 +148,22 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { } } test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc" || { + mkdir -p in2 echo 000000000000000000000000 > in/in2 echo 111 > in/in3 - mkdir -p in2 - ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? - CNT=`ls in2/* 2>/dev/null | wc -l` - case "$CNT" in - *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; - *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" - CODE=1 - ;; - esac - rm -f in2/in* + test "$OS" = "Darwin" && { + $ECHO "$GREY[*] afl-cmin not available on macOS, cannot test afl-cmin" + } || { + ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? + CNT=`ls in2/* 2>/dev/null | wc -l` + case "$CNT" in + *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" + CODE=1 + ;; + esac + rm -f in2/in* + } export AFL_QUIET=1 if type bash >/dev/null ; then { ../afl-cmin.bash -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null @@ -191,7 +197,8 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { for I in char short int long "long long"; do for BITS in 8 16 32 64; do bin="$testcase-split-$I-$BITS.compcov" - AFL_LLVM_INSTRUMENT=AFL AFL_DEBUG=1 AFL_LLVM_LAF_SPLIT_COMPARES_BITW=$BITS AFL_LLVM_LAF_SPLIT_COMPARES=1 ../afl-clang-fast -fsigned-char -DINT_TYPE="$I" -o "$bin" "$testcase" > test.out 2>&1; + #AFL_LLVM_INSTRUMENT=AFL + AFL_DEBUG=1 AFL_LLVM_LAF_SPLIT_COMPARES_BITW=$BITS AFL_LLVM_LAF_SPLIT_COMPARES=1 ../afl-clang-fast -fsigned-char -DINT_TYPE="$I" -o "$bin" "$testcase" > test.out 2>&1; if ! test -e "$bin"; then cat test.out $ECHO "$RED[!] llvm_mode laf-intel/compcov integer splitting failed! ($testcase with type $I split to $BITS)!"; @@ -257,13 +264,12 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { } rm -f test-compcov test.out instrumentlist.txt AFL_LLVM_CMPLOG=1 ../afl-clang-fast -o test-cmplog test-cmplog.c > /dev/null 2>&1 - ../afl-clang-fast -o test-c test-cmplog.c > /dev/null 2>&1 test -e test-cmplog && { $ECHO "$GREY[*] running afl-fuzz for llvm_mode cmplog, this will take approx 10 seconds" { mkdir -p in echo 00000000000000000000000000000000 > in/in - AFL_BENCH_UNTIL_CRASH=1 ../afl-fuzz -m none -V30 -i in -o out -c./test-cmplog -- ./test-c >>errors 2>&1 + AFL_BENCH_UNTIL_CRASH=1 ../afl-fuzz -Z -l 3 -m none -V30 -i in -o out -c 0 -- ./test-cmplog >>errors 2>&1 } >>errors 2>&1 test -n "$( ls out/default/crashes/id:000000* out/default/hangs/id:000000* 2>/dev/null )" && { $ECHO "$GREEN[+] afl-fuzz is working correctly with llvm_mode cmplog" @@ -278,7 +284,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { $ECHO "$YELLOW[-] we cannot test llvm_mode cmplog because it is not present" INCOMPLETE=1 } - rm -rf errors test-cmplog test-c in core.* + rm -rf errors test-cmplog in core.* ../afl-clang-fast -o test-persistent ../utils/persistent_mode/persistent_demo.c > /dev/null 2>&1 test -e test-persistent && { echo foo | AFL_QUIET=1 ../afl-showmap -m ${MEM_LIMIT} -o /dev/null -q -r ./test-persistent && { diff --git a/test/test-nyx-mode.sh b/test/test-nyx-mode.sh new file mode 100755 index 00000000..6de63f1b --- /dev/null +++ b/test/test-nyx-mode.sh @@ -0,0 +1,79 @@ +#!/bin/sh + +. ./test-pre.sh + +$ECHO "$BLUE[*] Testing: nyx_mode" + +test "$CI" = "true" && { + $ECHO "$YELLOW[-] nyx_mode cannot be tested in the Github CI, skipping ..." + exit 0 +} + +unset AFL_CC + +test -e ../libnyx.so && { + ../afl-cc -o test-instr ../test-instr.c > errors 2>&1 + test -e test-instr && { + { + rm -rf nyx-test in out + $ECHO "$GREY[*] running nyx_packer" + python3 ../nyx_mode/packer/packer/nyx_packer.py \ + ./test-instr \ + nyx-test \ + afl \ + instrumentation \ + --fast_reload_mode \ + --purge > /dev/null 2>&1 + + test -e nyx-test/test-instr && { + + $ECHO "$GREY[*] running nyx_config_gen" + python3 ../nyx_mode/packer/packer/nyx_config_gen.py nyx-test Kernel > /dev/null 2>&1 + + test -e nyx-test/config.ron && { + sudo modprobe -r kvm-intel + sudo modprobe -r kvm + sudo modprobe kvm enable_vmware_backdoor=y + sudo modprobe kvm-intel + #cat /sys/module/kvm/parameters/enable_vmware_backdoor + + mkdir -p in + echo 00000 > in/in + $ECHO "$GREY[*] running afl-fuzz for nyx_mode, this will take approx 10 seconds" + { + AFL_DEBUG=1 ../afl-fuzz -i in -o out -V05 -X -- ./nyx-test >>errors 2>&1 + } >>errors 2>&1 + test -n "$( ls out/default/queue/id:000002* 2>/dev/null )" && { + $ECHO "$GREEN[+] afl-fuzz is working correctly with nyx_mode" + RUNTIME=`grep execs_done out/default/fuzzer_stats | awk '{print$3}'` + rm -rf errors nyx-test test-instr in out + } || { + echo CUT------------------------------------------------------------------CUT + cat errors + echo CUT------------------------------------------------------------------CUT + $ECHO "$RED[!] afl-fuzz is not working correctly with nyx_mode" + CODE=1 + } + } || { + $ECHO "$RED[!] nyx_packer failed, likely install requirements not met." + CODE=1 + } + } || { + $ECHO "$RED[!] nyx_packer failed, likely install requirements not met." + CODE=1 + } + #rm -rf test-instr in out errors nyx-test + } + } || { + echo CUT------------------------------------------------------------------CUT + cat errors + echo CUT------------------------------------------------------------------CUT + $ECHO "$RED[!] afl-cc compilation of test targets failed - what is going on??" + CODE=1 + } +} || { + $ECHO "$YELLOW[-] nyx_mode is not compiled, cannot test" + INCOMPLETE=1 +} + +. ./test-post.sh diff --git a/test/test-pre.sh b/test/test-pre.sh index 1ca9dfb5..ce996415 100755 --- a/test/test-pre.sh +++ b/test/test-pre.sh @@ -20,7 +20,7 @@ echo foobar | grep -qE 'asd|oob' 2>/dev/null || { echo Error: grep command does test -e ./test-all.sh || cd $(dirname $0) || exit 1 test -e ./test-all.sh || { echo Error: you must be in the test/ directory ; exit 1 ; } export AFL_PATH=`pwd`/.. -export AFL_NO_AFFINITY=1 # workaround for travis that fails for no avail cores +export AFL_TRY_AFFINITY=1 # workaround for travis that fails for no avail cores echo 1 > test.1 echo 1 > test.2 diff --git a/test/test-qemu-mode.sh b/test/test-qemu-mode.sh index 9e268963..2ba81d02 100755 --- a/test/test-qemu-mode.sh +++ b/test/test-qemu-mode.sh @@ -63,7 +63,7 @@ test -e ../afl-qemu-trace && { { export AFL_PRELOAD=../libcompcov.so export AFL_COMPCOV_LEVEL=2 - ../afl-fuzz -m ${MEM_LIMIT} -V07 -Q -i in -o out -- ./test-compcov >>errors 2>&1 + AFL_NO_UI=1 ../afl-fuzz -V07 -Q -i in -o out -- ./test-compcov 2>&1 unset AFL_PRELOAD unset AFL_COMPCOV_LEVEL } >>errors 2>&1 @@ -88,7 +88,7 @@ test -e ../afl-qemu-trace && { test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc" -o "$SYS" = "aarch64" -o ! "${SYS%%arm*}" && { $ECHO "$GREY[*] running afl-fuzz for qemu_mode cmplog, this will take approx 10 seconds" { - ../afl-fuzz -m none -V07 -Q -c 0 -i in -o out -- ./test-compcov >>errors 2>&1 + ../afl-fuzz -V07 -Q -c 0 -l 3 -i in -o out -- ./test-compcov >>errors 2>&1 } >>errors 2>&1 test -n "$( ls out/default/queue/id:000001* 2>/dev/null )" && { $ECHO "$GREEN[+] afl-fuzz is working correctly with qemu_mode cmplog" diff --git a/test/unittests/unit_rand.c b/test/unittests/unit_rand.c index 1ad02a80..f89b2ab5 100644 --- a/test/unittests/unit_rand.c +++ b/test/unittests/unit_rand.c @@ -67,6 +67,7 @@ static void test_rand_below(void **state) { rand_set_seed(&afl, 1337); afl.fsrv.dev_urandom_fd = open("/dev/urandom", O_RDONLY); + if (afl.fsrv.dev_urandom_fd < 0) { PFATAL("Unable to open /dev/urandom"); } assert(!(rand_below(&afl, 9000) > 9000)); assert_int_equal(rand_below(&afl, 1), 0); diff --git a/unicorn_mode/UNICORNAFL_VERSION b/unicorn_mode/UNICORNAFL_VERSION index 1c8e571f..da17452d 100644 --- a/unicorn_mode/UNICORNAFL_VERSION +++ b/unicorn_mode/UNICORNAFL_VERSION @@ -1 +1 @@ -f2cede37 +764b66b2 diff --git a/unicorn_mode/build_unicorn_support.sh b/unicorn_mode/build_unicorn_support.sh index d3d16ad5..097a2dc9 100755 --- a/unicorn_mode/build_unicorn_support.sh +++ b/unicorn_mode/build_unicorn_support.sh @@ -14,7 +14,7 @@ # <andreafioraldi@gmail.com> # # Copyright 2017 Battelle Memorial Institute. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -198,10 +198,12 @@ $MAKECMD -j1 || exit 1 echo "[+] Build process successful!" echo "[*] Installing Unicorn python bindings..." +XOPT= +$PYTHONBIN -m pip install --help 2>/dev/null | grep -q break-system-packages && XOPT=--break-system-packages cd unicorn/bindings/python || exit 1 if [ -z "$VIRTUAL_ENV" ]; then echo "[*] Info: Installing python unicornafl using --user" - THREADS=$CORES $PYTHONBIN -m pip install --user --force .|| exit 1 + THREADS=$CORES $PYTHONBIN -m pip install --user $XOPT --force .|| exit 1 else echo "[*] Info: Installing python unicornafl to virtualenv: $VIRTUAL_ENV" THREADS=$CORES $PYTHONBIN -m pip install --force .|| exit 1 @@ -211,7 +213,7 @@ echo "[*] Installing Unicornafl python bindings..." cd bindings/python || exit 1 if [ -z "$VIRTUAL_ENV" ]; then echo "[*] Info: Installing python unicornafl using --user" - THREADS=$CORES $PYTHONBIN -m pip install --user --force .|| exit 1 + THREADS=$CORES $PYTHONBIN -m pip install --user $XOPT --force .|| exit 1 else echo "[*] Info: Installing python unicornafl to virtualenv: $VIRTUAL_ENV" THREADS=$CORES $PYTHONBIN -m pip install --force .|| exit 1 diff --git a/unicorn_mode/helper_scripts/unicorn_dumper_gdb.py b/unicorn_mode/helper_scripts/unicorn_dumper_gdb.py index 1ac4c9f3..a202ac0e 100644 --- a/unicorn_mode/helper_scripts/unicorn_dumper_gdb.py +++ b/unicorn_mode/helper_scripts/unicorn_dumper_gdb.py @@ -89,8 +89,8 @@ def dump_arch_info(): def dump_regs(): reg_state = {} - for reg in current_arch.all_registers: - reg_val = get_register(reg) + for reg in gef.arch.registers: + reg_val = gef.arch.register(reg) reg_state[reg.strip().strip("$")] = reg_val return reg_state @@ -101,7 +101,9 @@ def dump_process_memory(output_dir): final_segment_list = [] # GEF: - vmmap = get_process_maps() + vmmap = gef.memory.maps + memory = GefMemoryManager() + if not vmmap: print("No address mapping information found") return final_segment_list @@ -126,7 +128,7 @@ def dump_process_memory(output_dir): if entry.is_readable() and not "(deleted)" in entry.path: try: # Compress and dump the content to a file - seg_content = read_memory(entry.page_start, entry.size) + seg_content = memory.read(entry.page_start, entry.size) if seg_content == None: print( "Segment empty: @0x{0:016x} (size:UNKNOWN) {1}".format( diff --git a/unicorn_mode/helper_scripts/unicorn_dumper_pwndbg.py b/unicorn_mode/helper_scripts/unicorn_dumper_pwndbg.py index eccbc8bf..76eaf54f 100644 --- a/unicorn_mode/helper_scripts/unicorn_dumper_pwndbg.py +++ b/unicorn_mode/helper_scripts/unicorn_dumper_pwndbg.py @@ -40,10 +40,10 @@ import gdb pwndbg_loaded = False try: - import pwndbg.arch - import pwndbg.regs - import pwndbg.vmmap - import pwndbg.memory + import pwndbg.gdblib.arch + import pwndbg.gdblib.regs + import pwndbg.gdblib.vmmap + import pwndbg.gdblib.memory pwndbg_loaded = True @@ -64,7 +64,7 @@ INDEX_FILE_NAME = "_index.json" def map_arch(): - arch = pwndbg.arch.current # from PWNDBG + arch = pwndbg.gdblib.arch.current # from PWNDBG if "x86_64" in arch or "x86-64" in arch: return "x64" elif "x86" in arch or "i386" in arch: @@ -74,9 +74,9 @@ def map_arch(): elif "aarch64_be" in arch: return "arm64be" elif "arm" in arch: - cpsr = pwndbg.regs["cpsr"] + cpsr = pwndbg.gdblib.regs["cpsr"] # check endianess - if pwndbg.arch.endian == "big": + if pwndbg.gdblib.arch.endian == "big": # check for THUMB mode if cpsr & (1 << 5): return "armbethumb" @@ -89,7 +89,7 @@ def map_arch(): else: return "armle" elif "mips" in arch: - if pwndbg.arch.endian == "little": + if pwndbg.gdblib.arch.endian == "little": return "mipsel" else: return "mips" @@ -109,14 +109,16 @@ def dump_arch_info(): def dump_regs(): reg_state = {} - for reg in pwndbg.regs.all: - reg_val = pwndbg.regs[reg] + for reg in pwndbg.gdblib.regs.all: + reg_val = pwndbg.gdblib.regs[reg] + if reg_val is None: + continue # current dumper script looks for register values to be hex strings # reg_str = "0x{:08x}".format(reg_val) # if "64" in get_arch(): # reg_str = "0x{:016x}".format(reg_val) # reg_state[reg.strip().strip('$')] = reg_str - reg_state[reg.strip().strip("$")] = reg_val + reg_state[reg.strip().strip("$")] = int(reg_val) return reg_state @@ -125,7 +127,7 @@ def dump_process_memory(output_dir): final_segment_list = [] # PWNDBG: - vmmap = pwndbg.vmmap.get() + vmmap = pwndbg.gdblib.vmmap.get() # Pointer to end of last dumped memory segment segment_last_addr = 0x0 @@ -165,7 +167,7 @@ def dump_process_memory(output_dir): if entry.read and not "(deleted)" in entry.objfile: try: # Compress and dump the content to a file - seg_content = pwndbg.memory.read(start, end - start) + seg_content = pwndbg.gdblib.memory.read(start, end - start) if seg_content == None: print( "Segment empty: @0x{0:016x} (size:UNKNOWN) {1}".format( @@ -181,7 +183,7 @@ def dump_process_memory(output_dir): repr(seg_info["permissions"]), ) ) - compressed_seg_content = zlib.compress(str(seg_content)) + compressed_seg_content = zlib.compress(bytes(seg_content)) md5_sum = hashlib.md5(compressed_seg_content).hexdigest() + ".bin" seg_info["content_file"] = md5_sum diff --git a/unicorn_mode/helper_scripts/unicorn_loader.py b/unicorn_mode/helper_scripts/unicorn_loader.py index c48a7572..a83e7000 100644 --- a/unicorn_mode/helper_scripts/unicorn_loader.py +++ b/unicorn_mode/helper_scripts/unicorn_loader.py @@ -21,10 +21,10 @@ import zlib # Unicorn imports from unicornafl import * -from unicornafl.arm_const import * -from unicornafl.arm64_const import * -from unicornafl.x86_const import * -from unicornafl.mips_const import * +from unicorn.arm_const import * +from unicorn.arm64_const import * +from unicorn.x86_const import * +from unicorn.mips_const import * # If Capstone libraries are availible (only check once) try: @@ -87,9 +87,10 @@ class UnicornSimpleHeap(object): _uc = None # Unicorn engine instance to interact with _chunks = [] # List of all known chunks + _chunks_freed = [] # List of all freed chunks _debug_print = False # True to print debug information - def __init__(self, uc, debug_print=False): + def __init__(self, uc, debug_print=Falseļ¼ uaf_check=False): self._uc = uc self._debug_print = debug_print @@ -101,12 +102,23 @@ class UnicornSimpleHeap(object): # - Allocate at least 1 4k page of memory to make Unicorn happy # - Add guard pages at the start and end of the region total_chunk_size = UNICORN_PAGE_SIZE + ALIGN_PAGE_UP(size) + UNICORN_PAGE_SIZE + + if size == 0: + return 0 + # Gross but efficient way to find space for the chunk: chunk = None for addr in range(self.HEAP_MIN_ADDR, self.HEAP_MAX_ADDR, UNICORN_PAGE_SIZE): try: self._uc.mem_map(addr, total_chunk_size, UC_PROT_READ | UC_PROT_WRITE) chunk = self.HeapChunk(addr, total_chunk_size, size) + + if self.uaf_check: + for chunk_freed in self._chunks_freed: + if chunk_freed.is_buffer_in_chunk(chunk.data_addr, 1): + self._chunks_freed.remove(chunk_freed) + break + if self._debug_print: print( "Allocating 0x{0:x}-byte chunk @ 0x{1:016x}".format( @@ -148,6 +160,9 @@ class UnicornSimpleHeap(object): return new_chunk_addr def free(self, addr): + if addr == 0: + return False + for chunk in self._chunks: if chunk.is_buffer_in_chunk(addr, 1): if self._debug_print: @@ -157,9 +172,14 @@ class UnicornSimpleHeap(object): ) ) self._uc.mem_unmap(chunk.actual_addr, chunk.total_size) + + if self.uaf_check: + self._chunks_freed.append(chunk) + self._chunks.remove(chunk) return True - return False + # Freed an object that doesn't exist. Maybe 'dobule-free' or 'invalid free' vulnerability here. + self._uc.force_crash(UcError(UC_ERR_FETCH_UNMAPPED)) # Implements basic guard-page functionality def __check_mem_access(self, uc, access, address, size, value, user_data): @@ -179,6 +199,15 @@ class UnicornSimpleHeap(object): # Force a memory-based crash uc.force_crash(UcError(UC_ERR_READ_PROT)) + if self.uaf_check: + for chunk in self._chunks_freed: + if address >= chunk.actual_addr and ( + (address + size) <= (chunk.actual_addr + chunk.total_size) + ): + if chunk.is_buffer_in_chunk(address, size): + print("Use-after-free @ 0x{0:016x}".format(address)) + uc.force_crash(UcError(UC_ERR_FETCH_UNMAPPED)) + # --------------------------- # ---- Loading function diff --git a/unicorn_mode/unicornafl b/unicorn_mode/unicornafl -Subproject f2cede37a75bbd4a9b9438f0277727b5d462057 +Subproject 764b66b21cd4a8124a5b6c9cc98d1214b203719 diff --git a/utils/afl_network_proxy/GNUmakefile b/utils/afl_network_proxy/GNUmakefile index 7c8c22ff..47d9a7d3 100644 --- a/utils/afl_network_proxy/GNUmakefile +++ b/utils/afl_network_proxy/GNUmakefile @@ -10,6 +10,7 @@ PROGRAMS = afl-network-client afl-network-server HASH=\# CFLAGS += -Wno-pointer-sign +LDFLAGS += -ldl ifdef STATIC CFLAGS += -static diff --git a/utils/afl_network_proxy/afl-network-client.c b/utils/afl_network_proxy/afl-network-client.c index 0416f0f9..1f04dd87 100644 --- a/utils/afl_network_proxy/afl-network-client.c +++ b/utils/afl_network_proxy/afl-network-client.c @@ -4,7 +4,7 @@ Written by Marc Heuse <mh@mh-sec.de> - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_network_proxy/afl-network-server.c b/utils/afl_network_proxy/afl-network-server.c index 04309ada..c4a700f4 100644 --- a/utils/afl_network_proxy/afl-network-server.c +++ b/utils/afl_network_proxy/afl-network-server.c @@ -12,7 +12,7 @@ Dominik Maier <mail@dmnk.co> Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -173,6 +173,7 @@ static void set_up_environment(afl_forkserver_t *fsrv) { } out_file = alloc_printf("%s/.afl-input-temp-%u", use_dir, getpid()); + fsrv->out_file = out_file; } diff --git a/utils/afl_proxy/afl-proxy.c b/utils/afl_proxy/afl-proxy.c index 531a97a2..6cf47636 100644 --- a/utils/afl_proxy/afl-proxy.c +++ b/utils/afl_proxy/afl-proxy.c @@ -4,7 +4,7 @@ Written by Marc Heuse <mh@mh-sec.de> - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_untracer/Makefile b/utils/afl_untracer/Makefile index 14a09b41..264aebe5 100644 --- a/utils/afl_untracer/Makefile +++ b/utils/afl_untracer/Makefile @@ -3,11 +3,16 @@ ifdef DEBUG else OPT=-O3 endif +SYS = $(shell uname -s) +DL = +ifeq "$(SYS)" "Linux" + DL = -ldl +endif all: afl-untracer libtestinstr.so afl-untracer: afl-untracer.c - $(CC) $(OPT) -I../../include -g -o afl-untracer afl-untracer.c -ldl + $(CC) $(OPT) -I../../include -g -o afl-untracer afl-untracer.c $(DL) libtestinstr.so: libtestinstr.c $(CC) -g -O0 -fPIC -o libtestinstr.so -shared libtestinstr.c diff --git a/utils/afl_untracer/afl-untracer.c b/utils/afl_untracer/afl-untracer.c index a18e314e..e6a74518 100644 --- a/utils/afl_untracer/afl-untracer.c +++ b/utils/afl_untracer/afl-untracer.c @@ -4,7 +4,7 @@ Written by Marc Heuse <mh@mh-sec.de> - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -53,7 +53,9 @@ #include <pthread.h> #include <sys/mman.h> -#include <sys/shm.h> +#if !defined(__HAIKU__) + #include <sys/shm.h> +#endif #include <sys/wait.h> #include <sys/types.h> @@ -66,6 +68,9 @@ #include <sys/sysctl.h> #include <sys/user.h> #include <sys/procctl.h> +#elif defined(__HAIKU__) + #include <kernel/OS.h> + #include <kernel/image.h> #else #error "Unsupported platform" #endif @@ -232,6 +237,30 @@ void read_library_information(void) { } +#elif defined(__HAIKU__) + image_info ii; + int32 c = 0; + + while (get_next_image_info(0, &c, &ii) == B_OK) { + + liblist[liblist_cnt].name = (u8 *)strdup(ii.name); + liblist[liblist_cnt].addr_start = (u64)ii.text; + liblist[liblist_cnt].addr_end = (u64)((char *)ii.text + ii.text_size); + + if (debug) { + + fprintf(stderr, "%s:%lx (%lx-%lx)\n", liblist[liblist_cnt].name, + (unsigned long)(liblist[liblist_cnt].addr_end - + liblist[liblist_cnt].addr_start), + (unsigned long)liblist[liblist_cnt].addr_start, + (unsigned long)(liblist[liblist_cnt].addr_end - 1)); + + } + + liblist_cnt++; + + } + #endif } @@ -288,7 +317,7 @@ library_list_t *find_library(char *name) { #pragma GCC optimize("O0") void breakpoint(void) { - if (debug) fprintf(stderr, "Breakpoint function \"breakpoint\" reached.\n"); + if (debug) fprintf(stderr, "Breakpoint function \"breakpoint\" reached.\n"); } @@ -655,6 +684,9 @@ static void sigtrap_handler(int signum, siginfo_t *si, void *context) { #elif defined(__FreeBSD__) && defined(__LP64__) ctx->uc_mcontext.mc_rip -= 1; addr = ctx->uc_mcontext.mc_rip; +#elif defined(__HAIKU__) && defined(__x86_64__) + ctx->uc_mcontext.rip -= 1; + addr = ctx->uc_mcontext.rip; #else #error "Unsupported platform" #endif diff --git a/utils/afl_untracer/libtestinstr.c b/utils/afl_untracer/libtestinstr.c index b7afc325..0a98778a 100644 --- a/utils/afl_untracer/libtestinstr.c +++ b/utils/afl_untracer/libtestinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index 4e8f466d..9ffb2383 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -279,7 +279,9 @@ __attribute__((weak)) int main(int argc, char **argv) { */ - if (argc < 2 || strncmp(argv[1], "-h", 2) == 0) + if (argc < 2 || strncmp(argv[1], "-h", 2) == 0 || + strcmp(argv[1], "--help") == 0) { + printf( "============================== INFO ================================\n" "This binary is built for afl++.\n" @@ -290,12 +292,21 @@ __attribute__((weak)) int main(int argc, char **argv) { "afl-fuzz will run N iterations before re-spawning the process " "(default: " "INT_MAX)\n" + "You can also use AFL_FUZZER_LOOPCOUNT to set N\n" "For stdin input processing, pass '-' as single command line option.\n" "For file input processing, pass '@@' as single command line option.\n" "To use with afl-cmin or afl-cmin.bash pass '-' as single command line " "option\n" "===================================================================\n", argv[0], argv[0]); + if (argc == 2 && + (strncmp(argv[1], "-h", 2) == 0 || strcmp(argv[1], "--help") == 0)) { + + exit(0); + + } + + } return LLVMFuzzerRunDriver(&argc, &argv, LLVMFuzzerTestOneInput); @@ -369,6 +380,12 @@ __attribute__((weak)) int LLVMFuzzerRunDriver( } + if (getenv("AFL_FUZZER_LOOPCOUNT")) { + + N = atoi(getenv("AFL_FUZZER_LOOPCOUNT")); + + } + assert(N > 0); __afl_manual_init(); diff --git a/utils/argv_fuzzing/Makefile b/utils/argv_fuzzing/Makefile index 6786467a..ba977e5f 100644 --- a/utils/argv_fuzzing/Makefile +++ b/utils/argv_fuzzing/Makefile @@ -2,7 +2,7 @@ # american fuzzy lop++ - argvfuzz # -------------------------------- # -# Copyright 2019-2023 Kjell Braden <afflux@pentabarf.de> +# Copyright 2019-2024 Kjell Braden <afflux@pentabarf.de> # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/utils/argv_fuzzing/argvfuzz.c b/utils/argv_fuzzing/argvfuzz.c index 41eead0c..47383138 100644 --- a/utils/argv_fuzzing/argvfuzz.c +++ b/utils/argv_fuzzing/argvfuzz.c @@ -2,7 +2,7 @@ american fuzzy lop++ - LD_PRELOAD for fuzzing argv in binaries ------------------------------------------------------------ - Copyright 2019-2023 Kjell Braden <afflux@pentabarf.de> + Copyright 2019-2024 Kjell Braden <afflux@pentabarf.de> Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/bench/Makefile b/utils/bench/Makefile new file mode 100644 index 00000000..e7d2f3a1 --- /dev/null +++ b/utils/bench/Makefile @@ -0,0 +1,8 @@ +all: hash + +hash: hash.c + gcc -O3 -mavx2 -march=native -I../../include -o hash hash.c + +clean: + rm -f hash + diff --git a/utils/bench/README.md b/utils/bench/README.md new file mode 100644 index 00000000..772c117b --- /dev/null +++ b/utils/bench/README.md @@ -0,0 +1,2 @@ +# Internal AFL++ benchmarking + diff --git a/utils/bench/hash.c b/utils/bench/hash.c new file mode 100644 index 00000000..d4be0ab4 --- /dev/null +++ b/utils/bench/hash.c @@ -0,0 +1,53 @@ +#include <stdio.h> +#include <stdint.h> +#include <time.h> + +#define T1HA0_AESNI_AVAILABLE 1 +#define T1HA_USE_FAST_ONESHOT_READ 1 +#define T1HA_USE_INDIRECT_FUNCTIONS 1 +#define T1HA_IA32AES_NAME t1ha0_ia32aes +#include "t1ha0_ia32aes_b.h" + +#define XXH_INLINE_ALL +#include "xxhash.h" +#undef XXH_INLINE_ALL + +int main() { + + char *data = malloc(4097); + struct timespec start, end; + long long duration; + int i; + uint64_t res; + + clock_gettime(CLOCK_MONOTONIC, &start); + for (i = 0; i < 100000000; ++i) { + + res = XXH3_64bits(data, 4097); + memcpy(data + 16, (char *)&res, 8); + + } + + clock_gettime(CLOCK_MONOTONIC, &end); + duration = (end.tv_sec - start.tv_sec) * 1000000000LL + + (end.tv_nsec - start.tv_nsec); + printf("xxh3 duration: %lld ns\n", duration); + + memset(data, 0, 4097); + clock_gettime(CLOCK_MONOTONIC, &start); + for (i = 0; i < 100000000; ++i) { + + res = t1ha0_ia32aes(data, 4097); + memcpy(data + 16, (char *)&res, 8); + + } + + clock_gettime(CLOCK_MONOTONIC, &end); + duration = (end.tv_sec - start.tv_sec) * 1000000000LL + + (end.tv_nsec - start.tv_nsec); + printf("t1ha0_ia32aes duration: %lld ns\n", duration); + + return 0; + +} + diff --git a/utils/distributed_fuzzing/sync_script.sh b/utils/distributed_fuzzing/sync_script.sh index b22816f1..861b65c8 100755 --- a/utils/distributed_fuzzing/sync_script.sh +++ b/utils/distributed_fuzzing/sync_script.sh @@ -6,7 +6,7 @@ # Originally written by Michal Zalewski # # Copyright 2014 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/utils/dynamic_covfilter/README.md b/utils/dynamic_covfilter/README.md new file mode 100644 index 00000000..381e0855 --- /dev/null +++ b/utils/dynamic_covfilter/README.md @@ -0,0 +1,60 @@ +# Dynamic Instrumentation Filter + +Sometimes it can be beneficial to limit the instrumentation feedback to +specific code locations. It is possible to do so at compile-time by simply +not instrumenting any undesired locations. However, there are situations +where doing this dynamically without requiring a new build can be beneficial. +Especially when dealing with larger builds, it is much more convenient to +select the target code locations at runtime instead of doing so at build time. + +There are two ways of doing this in AFL++. Both approaches require a build of +AFL++ with `CODE_COVERAGE=1`, so make sure to build AFL++ first by invoking + +`CODE_COVERAGE=1 make` + +Once you have built AFL++, you can choose out of two approaches: + +## Simple Selection with `AFL_PC_FILTER` + +This approach requires a build with `AFL_INSTRUMENTATION=llvmnative` or +`llvmcodecov` as well as an AddressSanitizer build with debug information. + +By setting the environment variable `AFL_PC_FILTER` to a string, the runtime +symbolizer is enabled in the AFL++ runtime. At startup, the runtime will call +the `__sanitizer_symbolize_pc` API to resolve every PC in every loaded module. +The runtime then matches the result using `strstr` and disables the PC guard +if the symbolized PC does not contain the specified string. + +This approach has the benefit of being very easy to use. The downside is that +it causes significant startup delays with large binaries and that it requires +an AddressSanitizer build. + +This method has no additional runtime overhead after startup. + +## Selection using pre-symbolized data file with `AFL_PC_FILTER_FILE` + +To avoid large startup time delays, a specific module can be pre-symbolized +using the `make_symbol_list.py` script. This script outputs a sorted list of +functions with their respective relative offsets and lengths in the target +binary: + +`python3 make_symbol_list.py libxul.so > libxul.symbols.txt` + +The resulting list can be filtered, e.g. using grep: + +`grep -i "webgl" libxul.symbols.txt > libxul.webgl.symbols.txt` + +Finally, you can run with `AFL_PC_FILTER_FILE=libxul.webgl.symbols.txt` to +restrict instrumentation feedback to the given locations. This approach only +has a minimal startup time delay due to the implementation only using binary +search on the given file per PC rather than reading debug information for every +PC. It also works well with Nyx, where symbolizing is usually disabled for the +target process to avoid delays with frequent crashes. + +Similar to the previous method, This approach requires a build with +`AFL_INSTRUMENTATION=llvmnative` or `llvmcodecov` as well debug information. +However, it does not require the ASan runtime as it doesn't do the symbolizing +in process. Due to the way it maps PCs to symbols, it is less accurate when it +comes to includes and inlines (it assumes all PCs within a function belong to +that function and originate from the same file). For most purposes, this should +be a reasonable simplification to quickly process even the largest binaries. diff --git a/utils/dynamic_covfilter/make_symbol_list.py b/utils/dynamic_covfilter/make_symbol_list.py new file mode 100644 index 00000000..d1dd6ab3 --- /dev/null +++ b/utils/dynamic_covfilter/make_symbol_list.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Written by Christian Holler <decoder at mozilla dot com> + +import json +import os +import sys +import subprocess + +if len(sys.argv) != 2: + print("Usage: %s binfile" % os.path.basename(sys.argv[0])) + sys.exit(1) + +binfile = sys.argv[1] + +addr2len = {} +addrs = [] + +output = subprocess.check_output(["objdump", "-t", binfile]).decode("utf-8") +for line in output.splitlines(): + line = line.replace("\t", " ") + components = [x for x in line.split(" ") if x] + if not components: + continue + try: + start_addr = int(components[0], 16) + except ValueError: + continue + + # Length has variable position in objdump output + length = None + for comp in components[1:]: + if len(comp) == 16: + try: + length = int(comp, 16) + break + except: + continue + + if length is None: + print("ERROR: Couldn't determine function section length: %s" % line) + + func = components[-1] + + addrs.append(start_addr) + addr2len[str(hex(start_addr))] = str(length) + +# The search implementation in the AFL runtime expects everything sorted. +addrs.sort() +addrs = [str(hex(addr)) for addr in addrs] + +# We symbolize in one go to speed things up with large binaries. +output = subprocess.check_output([ + "llvm-addr2line", + "--output-style=JSON", + "-f", "-C", "-a", "-e", + binfile], + input="\n".join(addrs).encode("utf-8")).decode("utf-8") + +output = output.strip().splitlines() +for line in output: + output = json.loads(line) + if "Symbol" in output and output["Address"] in addr2len: + final_output = [ + output["Address"], + addr2len[output["Address"]], + os.path.basename(output["ModuleName"]), + output["Symbol"][0]["FileName"], + output["Symbol"][0]["FunctionName"] + ] + print("\t".join(final_output)) diff --git a/utils/libdislocator/libdislocator.so.c b/utils/libdislocator/libdislocator.so.c index 1cd7abc6..b80be1a1 100644 --- a/utils/libdislocator/libdislocator.so.c +++ b/utils/libdislocator/libdislocator.so.c @@ -6,7 +6,7 @@ Originally written by Michal Zalewski Copyright 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/libtokencap/libtokencap.so.c b/utils/libtokencap/libtokencap.so.c index 299056ab..cc499150 100644 --- a/utils/libtokencap/libtokencap.so.c +++ b/utils/libtokencap/libtokencap.so.c @@ -6,7 +6,7 @@ Originally written by Michal Zalewski Copyright 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -55,7 +55,7 @@ #elif defined __HAIKU__ #include <kernel/image.h> #elif defined __sun - /* For map addresses the old struct is enough */ +/* For map addresses the old struct is enough */ #include <sys/procfs.h> #include <limits.h> #endif @@ -81,7 +81,11 @@ void *(*__libc_memmem)(const void *haystack, size_t haystack_len, #define MAX_MAPPINGS 1024 -static struct mapping { void *st, *en; } __tokencap_ro[MAX_MAPPINGS]; +static struct mapping { + + void *st, *en; + +} __tokencap_ro[MAX_MAPPINGS]; static u32 __tokencap_ro_cnt; static u8 __tokencap_ro_loaded; @@ -164,7 +168,7 @@ static void __tokencap_load_mappings(void) { #elif defined __FreeBSD__ || defined __OpenBSD__ || defined __NetBSD__ #if defined __FreeBSD__ - int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, __tokencap_pid}; + int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, __tokencap_pid}; #elif defined __OpenBSD__ int mib[] = {CTL_KERN, KERN_PROC_VMMAP, __tokencap_pid}; #elif defined __NetBSD__ @@ -205,7 +209,7 @@ static void __tokencap_load_mappings(void) { #if defined __FreeBSD__ || defined __NetBSD__ #if defined __FreeBSD__ - size_t size = region->kve_structsize; + size_t size = region->kve_structsize; if (size == 0) break; #elif defined __NetBSD__ diff --git a/utils/persistent_mode/Makefile b/utils/persistent_mode/Makefile index e348c46c..498aa3f8 100644 --- a/utils/persistent_mode/Makefile +++ b/utils/persistent_mode/Makefile @@ -7,4 +7,4 @@ document: AFL_DONT_OPTIMIZE=1 ../../afl-clang-fast -D_AFL_DOCUMENT_MUTATIONS -o test-instr test-instr.c clean: - rm -f persistent_demo persistent_demo_new test-instr + rm -f persistent_demo persistent_demo_new persistent_demo_new_compat test-instr diff --git a/utils/persistent_mode/test-instr.c b/utils/persistent_mode/test-instr.c index 4ead6577..72e26e93 100644 --- a/utils/persistent_mode/test-instr.c +++ b/utils/persistent_mode/test-instr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/utils/qbdi_mode/build.sh b/utils/qbdi_mode/build.sh index 29fe0ee4..a92d81bd 100755 --- a/utils/qbdi_mode/build.sh +++ b/utils/qbdi_mode/build.sh @@ -52,6 +52,6 @@ ${compiler_prefix}${CC} -shared -o libdemo.so demo-so.c -w -g echo "[+] Building afl-fuzz for Android" # build afl-fuzz cd ../.. -${compiler_prefix}${CC} -DANDROID_DISABLE_FANCY=1 -O3 -funroll-loops -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign -I include/ -DAFL_PATH=\"/usr/local/lib/afl\" -DBIN_PATH=\"/usr/local/bin\" -DDOC_PATH=\"/usr/local/share/doc/afl\" -Wno-unused-function src/afl-fuzz*.c src/afl-common.c src/afl-sharedmem.c src/afl-forkserver.c src/afl-performance.c -o utils/qbdi_mode/afl-fuzz -ldl -lm -w +${compiler_prefix}${CC} -O3 -funroll-loops -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign -I include/ -DAFL_PATH=\"/usr/local/lib/afl\" -DBIN_PATH=\"/usr/local/bin\" -DDOC_PATH=\"/usr/local/share/doc/afl\" -Wno-unused-function src/afl-fuzz*.c src/afl-common.c src/afl-sharedmem.c src/afl-forkserver.c src/afl-performance.c -o utils/qbdi_mode/afl-fuzz -ldl -lm -w echo "[+] All done. Enjoy!" diff --git a/utils/replay_record/Makefile b/utils/replay_record/Makefile new file mode 100644 index 00000000..0d1cba92 --- /dev/null +++ b/utils/replay_record/Makefile @@ -0,0 +1,8 @@ +all: + test `grep '//[\s\t ]*#define[\s\t ]*AFL_PERSISTENT_RECORD' ../../include/config.h | wc -l` -eq 0 || (echo "AFL_PERSISTENT_RECORD must be enabled in config.h"; exit 1) + ../../afl-clang-fast -o persistent_demo_replay persistent_demo_replay.c + ${CC} -I ../../include -o persistent_demo_replay_compat persistent_demo_replay.c + ${CC} -g -I ../../include -DAFL_PERSISTENT_REPLAY_ARGPARSE -o persistent_demo_replay_argparse persistent_demo_replay.c + +clean: + rm -f persistent_demo_replay persistent_demo_replay_argparse persistent_demo_replay_compat diff --git a/utils/replay_record/README.md b/utils/replay_record/README.md new file mode 100644 index 00000000..6d72ca97 --- /dev/null +++ b/utils/replay_record/README.md @@ -0,0 +1,10 @@ +# AFL++ persistent record replay + +This persistent record replay demo showcases the `AFL_PERSISTENT_RECORD` replay functionality. + +The [Makefile](Makefile) will produce three binaries: + + persistent_demo_replay: uses afl-cc and makes use of the replay functionality included in the compiler runtime library + + persistent_demo_replay_compat: uses the [afl-record-compat.h](../../include/afl-record-compat.h) compatibility header to compile the same example without `afl-cc` + + persistent_demo_replay_argparse: makes use of `afl-record-compat.h`, and the Makefile defines `AFL_PERSISTENT_REPLAY_ARGPARSE` to test the replay functionality but parses the input file via a command-line argument (`@@`-style harness). + +For more information see [README.persistent_mode.md](../../instrumentation/README.persistent_mode.md). \ No newline at end of file diff --git a/utils/replay_record/persistent_demo_replay.c b/utils/replay_record/persistent_demo_replay.c new file mode 100644 index 00000000..6f6648f1 --- /dev/null +++ b/utils/replay_record/persistent_demo_replay.c @@ -0,0 +1,148 @@ +/* + american fuzzy lop++ - persistent mode example + -------------------------------------------- + + Originally written by Michal Zalewski + + Copyright 2015 Google Inc. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + + This file demonstrates the high-performance "persistent mode" that may be + suitable for fuzzing certain fast and well-behaved libraries, provided that + they are stateless or that their internal state can be easily reset + across runs. + + To make this work, the library and this shim need to be compiled in LLVM + mode using afl-clang-fast (other compiler wrappers will *not* work). + + */ + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <signal.h> +#include <string.h> +#include <limits.h> + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + #include <sys/stat.h> + #include <fcntl.h> +#endif + +/* this lets the source compile without afl-clang-fast/lto */ +#ifndef __AFL_FUZZ_TESTCASE_LEN + #include "afl-record-compat.h" +#endif + +__AFL_FUZZ_INIT(); + +/* Main entry point. */ + +/* To ensure checks are not optimized out it is recommended to disable + code optimization for the fuzzer harness main() */ +#pragma clang optimize off +#pragma GCC optimize("O0") + +int main(int argc, char **argv) { + + ssize_t len; /* how much input did we read? */ + unsigned char *buf; /* test case buffer pointer */ + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + int fd; + + if (argc < 2) { printf("Need an input file!"); } +#endif + + /* The number passed to __AFL_LOOP() controls the maximum number of + iterations before the loop exits and the program is allowed to + terminate normally. This limits the impact of accidental memory leaks + and similar hiccups. */ + + __AFL_INIT(); + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + buf = malloc(1000); +#else + buf = __AFL_FUZZ_TESTCASE_BUF; // this must be assigned before __AFL_LOOP! +#endif + + while (__AFL_LOOP(UINT_MAX)) { // increase if you have good stability + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + fd = open(argv[1], O_RDONLY); + len = read(fd, buf, 1000); + close(fd); +#else + len = __AFL_FUZZ_TESTCASE_LEN; // do not use the macro directly in a call! +#endif + + // fprintf(stderr, "input: %zd \"%s\"\n", len, buf); + + /* do we have enough data? */ + if (len < 8) continue; + + if (strcmp((char *)buf, "thisisateststring") == 0) printf("teststring\n"); + + if (buf[0] == 'f') { + + printf("one\n"); + if (buf[1] == 'o') { + + printf("two\n"); + if (buf[2] == 'o') { + + printf("three\n"); + if (buf[3] == '!') { + + printf("four\n"); + if (buf[4] == '!') { + + printf("five\n"); + if (buf[5] == '!') { + + printf("six\n"); + abort(); + + } else { + + if (buf[5] == 'O') { + + // hang + while (1) { + + continue; + + }; + + } + + } + + } + + } + + } + + } + + } + + /*** END PLACEHOLDER CODE ***/ + + } + + /* Once the loop is exited, terminate normally - AFL will restart the process + when this happens, with a clean slate when it comes to allocated memory, + leftover file descriptors, etc. */ + + return 0; + +} + diff --git a/utils/socket_fuzzing/socketfuzz.c b/utils/socket_fuzzing/socketfuzz.c index 3ec8383b..7497519e 100644 --- a/utils/socket_fuzzing/socketfuzz.c +++ b/utils/socket_fuzzing/socketfuzz.c @@ -23,7 +23,8 @@ #include <errno.h> #include <stdio.h> #include <poll.h> -//#include "logging.h" // switche from preeny_info() to fprintf(stderr, "Info: " +// #include "logging.h" // switched from preeny_info() to fprintf(stderr, "Info: +// " // // originals diff --git a/utils/target_intelligence/README.md b/utils/target_intelligence/README.md new file mode 100644 index 00000000..086c9e20 --- /dev/null +++ b/utils/target_intelligence/README.md @@ -0,0 +1,61 @@ +# Target Intelligence + +These are some ideas you can do so that your target that you are fuzzing can +give helpful feedback to AFL++. + +## Add to the AFL++ dictionary from your target + +For this you target must be compiled for CMPLOG (`AFL_LLVM_CMPLOG=1`). + +Add in your source code: + +``` +__attribute__((weak)) void __cmplog_rtn_hook_strn(u8 *ptr1, u8 *ptr2, u64 len); +__attribute__((weak)) void __cmplog_ins_hook1(uint8_t arg1, uint8_t arg2, uint8_t attr); +__attribute__((weak)) void __cmplog_ins_hook2(uint16_t arg1, uint16_t arg2, uint8_t attr); +__attribute__((weak)) void __cmplog_ins_hook4(uint32_t arg1, uint32_t arg2, uint8_t attr); +__attribute__((weak)) void __cmplog_ins_hook8(uint64_t arg1, uint64_t arg2, uint8_t attr); + +int in_your_function(...) { + + // to add two strings to the AFL++ dictionary: + if (__cmplog_rtn_hook_strn) + __cmplog_rtn_hook_strn(string1, length_of_string1, string2, length_of_string2); + + // to add two 32 bit integers to the AFL++ dictionary: + if (__cmplog_ins_hook4) + __cmplog_ins_hook4(first_32_bit_var, second_32_bit_var, 0); + +} +``` + +Note that this only makes sense if these values are in-depth processed in the +target in a way that AFL++ CMPLOG cannot uncover these, e.g. if these values +are transformed by a matrix computation. + +Fixed values are always better to give to afl-fuzz via a `-x dictionary`. + +## Add inputs to AFL++ dictionary from your target + +If for whatever reason you want your target to propose new inputs to AFL++, +then this is actually very easy. +The environment variable `AFL_CUSTOM_INFO_OUT` contains the output directory +of this run - including the fuzzer instance name (e.g. `default`), so if you +run `afl-fuzz -o out -S foobar`, the value would be `out/foobar`). + +To show afl-fuzz an input it should consider just do the following: + +1. create the directory `$AFL_CUSTOM_INFO_OUT/../target/queue` +2. create any new inputs you want afl-fuzz to notice in that directory with the + following naming convention: `id:NUMBER-OF-LENGTH-SIX-WITH-LEADING-ZEROES,whatever` + where that number has to be increasing. + e.g.: +``` + id:000000,first_file + id:000001,second_file + id:000002,third_file + etc. +``` + +Note that this will not work in nyx_mode because afl-fuzz cannot see inside the +virtual machine. |
