diff options
| -rw-r--r-- | README.md | 2 | ||||
| -rwxr-xr-x | afl-whatsup | 121 | ||||
| -rw-r--r-- | benchmark/COMPARISON | 4 | ||||
| -rwxr-xr-x | benchmark/benchmark.sh | 42 | ||||
| -rw-r--r-- | custom_mutators/aflpp_tritondse/aflpp_tritondse.py | 2 | ||||
| -rw-r--r-- | custom_mutators/examples/README.md | 3 | ||||
| -rw-r--r-- | custom_mutators/examples/elf_header_mutator.c | 679 | ||||
| -rw-r--r-- | docs/Changelog.md | 15 | ||||
| -rw-r--r-- | docs/FAQ.md | 40 | ||||
| -rw-r--r-- | docs/env_variables.md | 13 | ||||
| -rw-r--r-- | include/afl-fuzz.h | 12 | ||||
| -rw-r--r-- | include/config.h | 2 | ||||
| -rw-r--r-- | include/envs.h | 1 | ||||
| -rw-r--r-- | src/afl-fuzz-bitmap.c | 2 | ||||
| -rw-r--r-- | src/afl-fuzz-queue.c | 48 | ||||
| -rw-r--r-- | src/afl-fuzz-state.c | 7 | ||||
| -rw-r--r-- | src/afl-fuzz.c | 11 |
17 files changed, 927 insertions, 77 deletions
diff --git a/README.md b/README.md index 951efe59..322ebcf2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Release version: [4.08c](https://github.com/AFLplusplus/AFLplusplus/releases) -GitHub version: 4.08c +GitHub version: 4.09a Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) diff --git a/afl-whatsup b/afl-whatsup index 6f29ab24..bbb73e47 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -18,49 +18,78 @@ # instances of afl-fuzz. # -echo "$0 status check tool for afl-fuzz by Michal Zalewski" -echo test "$1" = "-h" -o "$1" = "-hh" && { + echo "$0 status check tool for afl-fuzz by Michal Zalewski" + echo echo "Usage: $0 [-s] [-d] afl_output_directory" echo echo Options: - echo " -s - skip details and output summary results only" echo " -d - include dead fuzzer stats" + echo " -m - just show minimal stats" + echo " -n - no color output" + echo " -s - skip details and output summary results only" echo exit 1 } -unset SUMMARY_ONLY +unset MINIMAL_ONLY +unset NO_COLOR unset PROCESS_DEAD +unset SUMMARY_ONLY +unset RED +unset GREEN +unset YELLOW +unset BLUE +unset NC +unset RESET -while [ "$1" = "-s" -o "$1" = "-d" ]; do +if [ -z "$TERM" ]; then export TERM=vt220; fi - if [ "$1" = "-s" ]; then - SUMMARY_ONLY=1 - fi +while [ "$1" = "-d" -o "$1" = "-m" -o "$1" = "-n" -o "$1" = "-s" ]; do if [ "$1" = "-d" ]; then PROCESS_DEAD=1 fi + if [ "$1" = "-m" ]; then + MINIMAL_ONLY=1 + fi + + if [ "$1" = "-n" ]; then + NO_COLOR=1 + fi + + if [ "$1" = "-s" ]; then + SUMMARY_ONLY=1 + fi + shift done DIR="$1" -if [ "$DIR" = "" ]; then +if [ "$DIR" = "" -o "$DIR" = "-h" -o "$DIR" = "--help" ]; then - echo "Usage: $0 [-s] [-d] afl_output_directory" 1>&2 + echo "$0 status check tool for afl-fuzz by Michal Zalewski" 1>&2 + echo 1>&2 + echo "Usage: $0 [-d] [-m] [-n] [-s] afl_output_directory" 1>&2 echo 1>&2 echo Options: 1>&2 - echo " -s - skip details and output summary results only" 1>&2 echo " -d - include dead fuzzer stats" 1>&2 + echo " -m - just show minimal stats" 1>&2 + echo " -n - no color output" 1>&2 + echo " -s - skip details and output summary results only" 1>&2 echo 1>&2 exit 1 fi +if [ -z "$MINIMAL_ONLY" ]; then + echo "$0 status check tool for afl-fuzz by Michal Zalewski" + echo +fi + cd "$DIR" || exit 1 if [ -d queue ]; then @@ -70,12 +99,16 @@ if [ -d queue ]; then fi -RED=`tput setaf 9 1 1 2>/dev/null` -GREEN=`tput setaf 2 1 1 2>/dev/null` -BLUE=`tput setaf 4 1 1 2>/dev/null` -YELLOW=`tput setaf 11 1 1 2>/dev/null` -NC=`tput sgr0` -RESET="$NC" +BC=`which bc 2>/dev/null` + +if [ -z "$NO_COLOR" ]; then + RED=`tput setaf 9 1 1 2>/dev/null` + GREEN=`tput setaf 2 1 1 2>/dev/null` + BLUE=`tput setaf 4 1 1 2>/dev/null` + YELLOW=`tput setaf 11 1 1 2>/dev/null` + NC=`tput sgr0` + RESET="$NC" +fi CUR_TIME=`date +%s` @@ -91,6 +124,7 @@ TOTAL_CRASHES=0 TOTAL_HANGS=0 TOTAL_PFAV=0 TOTAL_PENDING=0 +TOTAL_COVERAGE= # Time since last find / crash / hang, formatted as string FMT_TIME="0 days 0 hours" @@ -147,6 +181,13 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do RUN_UNIX=$run_time RUN_DAYS=$((RUN_UNIX / 60 / 60 / 24)) RUN_HRS=$(((RUN_UNIX / 60 / 60) % 24)) + COVERAGE=$(echo $bitmap_cvg|tr -d %) + if [ -n "$TOTAL_COVERAGE" -a -n "$B" ]; then + if [ "$(echo "$TOTAL_COVERAGE < $COVERAGE" | bc)" -eq 1 ]; then + TOTAL_COVERAGE=$COVERAGE + fi + fi + if [ -z "$TOTAL_COVERAGE" ]; then TOTAL_COVERAGE=$COVERAGE ; fi test -n "$cycles_wo_finds" && { test -z "$FIRST" && TOTAL_WCOP="${TOTAL_WCOP}/" @@ -225,13 +266,21 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do echo " last_find : $FMT_FIND" echo " last_crash : $FMT_CRASH" - echo " last_hang : $FMT_HANG" - echo " cycles_wo_finds : $FMT_CWOP" + if [ -z "$MINIMAL_ONLY" ]; then + echo " last_hang : $FMT_HANG" + echo " cycles_wo_finds : $FMT_CWOP" + fi + echo " coverage : $COVERAGE%" + + if [ -z "$MINIMAL_ONLY" ]; then + + CPU_USAGE=$(ps aux | grep $fuzzer_pid | grep -v grep | awk '{print $3}') + MEM_USAGE=$(ps aux | grep $fuzzer_pid | grep -v grep | awk '{print $4}') - CPU_USAGE=$(ps aux | grep $fuzzer_pid | grep -v grep | awk '{print $3}') - MEM_USAGE=$(ps aux | grep $fuzzer_pid | grep -v grep | awk '{print $4}') + echo " cpu usage $CPU_USAGE%, memory usage $MEM_USAGE%" + + fi - echo " cpu usage $CPU_USAGE%, memory usage $MEM_USAGE%" echo " cycles $((cycles_done + 1)), lifetime speed $EXEC_SEC execs/sec, items $cur_item/$corpus_count (${PATH_PERC}%)" if [ "$saved_crashes" = "0" ]; then @@ -283,7 +332,10 @@ fi echo "Summary stats" echo "=============" -echo +if [ -z "$SUMMARY_ONLY" -o -z "$MINIMAL_ONLY" ]; then + echo +fi + echo " Fuzzers alive : $ALIVE_CNT" if [ ! "$DEAD_CNT" = "0" ]; then @@ -291,20 +343,29 @@ if [ ! "$DEAD_CNT" = "0" ]; then fi echo " Total run time : $FMT_TIME" -echo " Total execs : $FMT_EXECS" -echo " Cumulative speed : $TOTAL_EPS execs/sec" +if [ -z "$MINIMAL_ONLY" ]; then + echo " Total execs : $FMT_EXECS" + echo " Cumulative speed : $TOTAL_EPS execs/sec" +fi if [ "$ALIVE_CNT" -gt "0" ]; then echo " Average speed : $((TOTAL_EPS / ALIVE_CNT)) execs/sec" fi -echo " Pending items : $TOTAL_PFAV faves, $TOTAL_PENDING total" +if [ -z "$MINIMAL_ONLY" ]; then + echo " Pending items : $TOTAL_PFAV faves, $TOTAL_PENDING total" +fi -if [ "$ALIVE_CNT" -gt "1" ]; then - echo " Pending per fuzzer : $((TOTAL_PFAV/ALIVE_CNT)) faves, $((TOTAL_PENDING/ALIVE_CNT)) total (on average)" +if [ "$ALIVE_CNT" -gt "1" -o -n "$MINIMAL_ONLY" ]; then + if [ "$ALIVE_CNT" -gt "0" ]; then + echo " Pending per fuzzer : $((TOTAL_PFAV/ALIVE_CNT)) faves, $((TOTAL_PENDING/ALIVE_CNT)) total (on average)" + fi fi +echo " Coverage reached : ${TOTAL_COVERAGE}%" echo " Crashes saved : $TOTAL_CRASHES" -echo " Hangs saved : $TOTAL_HANGS" -echo "Cycles without finds : $TOTAL_WCOP" +if [ -z "$MINIMAL_ONLY" ]; then + echo " Hangs saved : $TOTAL_HANGS" + echo "Cycles without finds : $TOTAL_WCOP" +fi echo " Time without finds : $TOTAL_LAST_FIND" echo diff --git a/benchmark/COMPARISON b/benchmark/COMPARISON new file mode 100644 index 00000000..55ab94b4 --- /dev/null +++ b/benchmark/COMPARISON @@ -0,0 +1,4 @@ +CPU | Mz | exec/s | afl-*-config | +========================================|======|========|==============| +CPU 12th Gen Intel(R) Core(TM) i7-1270P | 4200 | 12750 | both | +AMD EPYC 7282 16-Core Processor | 3190 | 10060 | both | diff --git a/benchmark/benchmark.sh b/benchmark/benchmark.sh new file mode 100755 index 00000000..3318adce --- /dev/null +++ b/benchmark/benchmark.sh @@ -0,0 +1,42 @@ +#!/bin/sh +test -x ../afl-fuzz -a -x ../afl-cc -a -e ../SanitizerCoveragePCGUARD.so || { + echo Error: you need to compile AFL++ first, we need afl-fuzz, afl-clang-fast and SanitizerCoveragePCGUARD.so built. + exit 1 +} + +echo Preparing environment + +env | grep AFL_ | sed 's/=.*//' | while read e; do + unset $e +done + +AFL_PATH=`pwd`/.. +export PATH=$AFL_PATH:$PATH + +AFL_LLVM_INSTRUMENT=PCGUARD afl-cc -o test-instr ../test-instr.c > afl.log 2>&1 || { + echo Error: afl-cc is unable to compile + exit 1 +} + +{ +mkdir in +dd if=/dev/zero of=in/in.txt bs=10K count=1 +} > /dev/null 2>&1 + +echo Ready, starting benchmark - this will take approx 20-30 seconds ... + +AFL_DISABLE_TRIM=1 AFL_NO_UI=1 AFL_TRY_AFFINITY=1 AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_BENCH_JUST_ONE=1 time afl-fuzz -i in -o out -s 123 -D ./test-instr >> afl.log 2>&1 + +echo Analysis: + +CPUID=$(grep 'try binding to' afl.log | tail -n 1 | sed 's/.*#//' | sed 's/\..*//') +grep 'model name' /proc/cpuinfo | head -n 1 | sed 's/.*:/ CPU:/' +test -n "$CPUID" && grep -E '^processor|^cpu MHz' /proc/cpuinfo | grep -A1 -w "$CPUID" | grep 'cpu MHz' | head -n 1 | sed 's/.*:/ Mhz:/' +test -z "$CPUID" && grep 'cpu MHz' /proc/cpuinfo | head -n 1 | sed 's/.*:/ Mhz:/' +grep execs_per_sec out/default/fuzzer_stats | sed 's/.*:/ execs\/s:/' + +echo +echo "Comparison: (note that values can change by 10-15% per run)" +cat COMPARISON + +rm -rf in out test-instr afl.log diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py index 58b506b6..58739696 100644 --- a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -164,7 +164,7 @@ def init(seed): format = SeedFormat.COMPOSITE # Now set up TritonDSE config = Config(coverage_strategy = CoverageStrategy.PATH, - debug = is_debug, + # debug = is_debug, pipe_stdout = is_debug, pipe_stderr = is_debug, execution_timeout = 1, diff --git a/custom_mutators/examples/README.md b/custom_mutators/examples/README.md index 655f7a5e..112db243 100644 --- a/custom_mutators/examples/README.md +++ b/custom_mutators/examples/README.md @@ -33,3 +33,6 @@ like surgical_havoc_mutate() that allow to perform a randomly chosen mutation from a subset of the havoc mutations. If you do so, you have to specify -I /path/to/AFLplusplus/include when compiling. + +elf_header_mutator.c - example ELF header mutator based on + [LibGolf](https://github.com/xcellerator/libgolf/) diff --git a/custom_mutators/examples/elf_header_mutator.c b/custom_mutators/examples/elf_header_mutator.c new file mode 100644 index 00000000..b985257a --- /dev/null +++ b/custom_mutators/examples/elf_header_mutator.c @@ -0,0 +1,679 @@ +/* + AFL++ Custom Mutator for ELF Headers + Written by @echel0n <melih.sahin@protonmail.com> + based on libgolf.h by @xcellerator + $ gcc -O3 -fPIC -shared -o elf_mutator.so -I ~/AFLplusplus/include/ + */ +#include "afl-fuzz.h" +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <limits.h> +#include <linux/elf.h> + +/* EI_ABIVERSION isn't used anymore and elf.h defines EI_PAD to be 0x09 */ +#define EI_ABIVERSION 0x08 +#define EI_PAD 0x09 +/* Define the Architecture and ISA constants to match those in <linux/elf.h> */ +#define X86_64 EM_X86_64 +#define ARM32 EM_ARM +#define AARCH64 EM_AARCH64 +#define uchar unsigned char +#define DATA_SIZE 0x100 + +/* + * The ELF and Program headers are different sizes depending on 32- and 64-bit + * architectures + * taken from libgolf.h + */ +#define EHDR_T(x) Elf##x##_Ehdr +#define PHDR_T(x) Elf##x##_Phdr +#define EHDR(x) ehdr##x +#define PHDR(x) phdr##x +#define GET_EHDR(x) (&(elf_ptr->EHDR(x))); +#define GET_PHDR(x) (&(elf_ptr->PHDR(x))); +#define REF_EHDR(b, x) ((Elf##b##_Ehdr *)ehdr)->x +#define REF_PHDR(b, x) ((Elf##b##_Phdr *)phdr)->x +int ehdr_size; +int phdr_size; +/* + * This struct holds the bytes that will be executed, and the size. + */ +typedef struct text_segment { + + size_t text_size; + unsigned char *text_segment; + +} TextSegment; + +// example shellcode that exits +// taken from libgolf.h +unsigned char buf[] = {0xb0, 0x3c, 0x31, 0xff, 0x0f, 0x05}; + +/* + * This is the raw ELF file + * - EHDR(xx) is the ELF header + * - PHDR(xx) is the program header + * - text is the text segment + * - filename is the name of the golf'd binary + * - isa is the target architecture (X86_64, ARM32, AARCH64) + * taken from libgolf.h + */ +typedef struct rawbinary_t { + + EHDR_T(32) EHDR(32); + PHDR_T(32) PHDR(32); + EHDR_T(64) EHDR(64); + PHDR_T(64) PHDR(64); + TextSegment text; + char *filename; + int isa; + +} RawBinary; + +/* + * Copy an E_IDENT array into the corresponding fields in the ELF header + * Called by populate_ehdr() + * taken from libgolf.h + */ +int populate_e_ident(RawBinary *elf_ptr, unsigned char e_ident[]) { + + int i; + /* Depending on whether the target ISA is 32- or 64-bit, set e_ident */ + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + for (i = 0; i < EI_NIDENT; i++) + elf_ptr->EHDR(64).e_ident[i] = e_ident[i]; + break; + case ARM32: + for (i = 0; i < EI_NIDENT; i++) + elf_ptr->EHDR(32).e_ident[i] = e_ident[i]; + break; + default: + exit(1); + + } + + return 0; + +} + +/* + * Copy bytes from buf[] array into text_segment in ELF struct + * taken from libgolf.h + */ +int copy_text_segment(RawBinary *elf_ptr, unsigned char buf[], int text_size) { + + int i; + + /* Set size of text segment and allocate the buffer */ + elf_ptr->text.text_size = text_size; + elf_ptr->text.text_segment = + malloc(elf_ptr->text.text_size * sizeof(unsigned char)); + + /* Copy the bytes into the text segment buffer */ + for (i = 0; i < elf_ptr->text.text_size; i++) { + + elf_ptr->text.text_segment[i] = buf[i]; + + } + +} + +/* + * Populate the ELF Header with sane values + * Returns a pointer to an EHDR struct + * taken from libgolf.h + */ +void *populate_ehdr(RawBinary *elf_ptr) { + + /* + * Set ehdr_size and phdr_size. Determined by whether target ISA is 32- or + * 64-bit. + */ + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + ehdr_size = sizeof(EHDR_T(64)); + phdr_size = sizeof(PHDR_T(64)); + break; + case ARM32: + ehdr_size = sizeof(EHDR_T(32)); + phdr_size = sizeof(PHDR_T(32)); + break; + default: + exit(1); + + }; + + /* Start with the E_IDENT area at the top of the file */ + unsigned char e_ident[EI_NIDENT] = {0}; + + /* Magic Bytes */ + e_ident[EI_MAG0] = 0x7F; + e_ident[EI_MAG1] = 0x45; // E + e_ident[EI_MAG2] = 0x4C; // L + e_ident[EI_MAG3] = 0x46; // F + + /* + * EI_CLASS denotes the architecture: + * ELFCLASS32: 0x01 + * ELFCLASS64: 0x02 + */ + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + e_ident[EI_CLASS] = ELFCLASS64; + break; + case ARM32: + e_ident[EI_CLASS] = ELFCLASS32; + break; + default: + exit(1); + + } + + /* + * EI_DATA denotes the endianness: + * ELFDATA2LSB: 0x01 + * ELFDATA2MSB: 0x02 + */ + e_ident[EI_DATA] = ELFDATA2LSB; + + /* EI_VERSION is always 0x01 */ + e_ident[EI_VERSION] = EV_CURRENT; + + /* + * EI_OSABI defines the target OS. Ignored by most modern ELF parsers. + */ + e_ident[EI_OSABI] = ELFOSABI_NONE; + + /* EI_ABIVERSION was for sub-classification. Un-defined since Linux 2.6 */ + e_ident[EI_ABIVERSION] = 0x00; + + /* EI_PAD is currently unused */ + e_ident[EI_PAD] = 0x00; + + /* Copy the E_IDENT section to the ELF struct */ + populate_e_ident(elf_ptr, e_ident); + + /* + * The remainder of the ELF header following E_IDENT follows. + * + * ehdr is a pointer to either an Elf32_Edhr, or Elf64_Ehdr struct. + */ + void *ehdr = NULL; + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + ehdr = (&(elf_ptr->EHDR(64))); + break; + case ARM32: + ehdr = (&(elf_ptr->EHDR(32))); + break; + default: + exit(1); + + } + + /* + * Depending on whether the ISA is 32- or 64-bit determines the size of + * many of the fields in the ELF Header. This switch case deals with it. + */ + switch (elf_ptr->isa) { + + // 64-Bit ISAs + case X86_64: + case AARCH64: + /* + * e_type specifies what kind of ELF file this is: + * ET_NONE: 0x00 // Unknown Type + * ET_REL: 0x01 // Relocatable + * ET_EXEC: 0x02 // Executable File + * ET_DYN: 0x03 // Shared Object + * ET_CORE: 0x04 // Core Dump + */ + REF_EHDR(64, e_type) = ET_EXEC; // 0x0002 + + /* e_machine specifies the target ISA */ + REF_EHDR(64, e_machine) = elf_ptr->isa; + + /* e_version is always set of 0x01 for the original ELF spec */ + REF_EHDR(64, e_version) = EV_CURRENT; // 0x00000001 + + /* + * e_entry is the memory address of the entry point + * Set by set_entry_point() after p_vaddr is set in the phdr + */ + REF_EHDR(64, e_entry) = 0x0; + + /* + * e_phoff points to the start of the program header, which + * immediately follows the ELF header + */ + REF_EHDR(64, e_phoff) = ehdr_size; + + /* e_shoff points to the start of the section header table */ + REF_EHDR(64, e_shoff) = 0x00; + + /* e_flags is architecture dependent */ + REF_EHDR(64, e_flags) = 0x0; + + /* e_ehsize contains the size of the ELF header */ + REF_EHDR(64, e_ehsize) = ehdr_size; + + /* e_phentsize is the size of the program header */ + REF_EHDR(64, e_phentsize) = phdr_size; + + /* + * e_phnum contains the number of entries in the program header + * e_phnum * e_phentsize = size of program header table + */ + REF_EHDR(64, e_phnum) = 0x1; + + /* e_shentsize contains the size of a section header entry */ + REF_EHDR(64, e_shentsize) = 0x0; + + /* + * e_shnum contains the number of entries in the section header + * e_shnum * e_shentsize = size of section header table + */ + REF_EHDR(64, e_shnum) = 0x0; + + /* + * e_shstrndx contains the index of the section header table that + * contains the section names + */ + REF_EHDR(64, e_shstrndx) = 0x0; + + break; + // 32-Bit ISAs + case ARM32: + /* + * e_type specifies what kind of ELF file this is: + * ET_NONE: 0x00 // Unknown Type + * ET_REL: 0x01 // Relocatable + * ET_EXEC: 0x02 // Executable File + * ET_DYN: 0x03 // Shared Object + * ET_CORE: 0x04 // Core Dump + */ + REF_EHDR(32, e_type) = ET_EXEC; // 0x0002 + + /* e_machine specifies the target ISA */ + REF_EHDR(32, e_machine) = elf_ptr->isa; + + /* e_version is always set of 0x01 for the original ELF spec */ + REF_EHDR(32, e_version) = EV_CURRENT; // 0x00000001 + + /* + * e_entry is the memory address of the entry point + * Set by set_entry_point() after p_vaddr is set in the phdr + */ + REF_EHDR(32, e_entry) = 0x0; + + /* + * e_phoff points to the start of the program header, which + * immediately follows the ELF header + */ + REF_EHDR(32, e_phoff) = ehdr_size; + + /* e_shoff points to the start of the section header table */ + REF_EHDR(32, e_shoff) = 0x0i; + + /* e_flags is architecture dependent */ + REF_EHDR(32, e_flags) = 0x0; + + /* e_ehsize contains the size of the ELF header */ + REF_EHDR(32, e_ehsize) = ehdr_size; + + /* e_phentsize is the size of the program header */ + REF_EHDR(32, e_phentsize) = phdr_size; + + /* + * e_phnum contains the number of entries in the program header + * e_phnum * e_phentsize = size of program header table + */ + REF_EHDR(32, e_phnum) = 0x1; + + /* e_shentsize contains the size of a section header entry */ + REF_EHDR(32, e_shentsize) = 0x0; + + /* + * e_shnum contains the number of entries in the section header + * e_shnum * e_shentsize = size of section header table + */ + REF_EHDR(32, e_shnum) = 0x0; + + /* + * e_shstrndx contains the index of the section header table that + * contains the section names + */ + REF_EHDR(32, e_shnum) = 0x0; + + break; + + } + + return ehdr; + +} + +/* + * Populate the program headers with sane values + * Returns a pointer to a PHDR struct + * taken from libgolf.h + */ +void *populate_phdr(RawBinary *elf_ptr) { + + /* + * All offsets are relative to the start of the program header (0x40) + * + * phdr is a pointer to either an Elf32_Phdr, or Elf64_Phdr struct. + */ + void *phdr = NULL; + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + phdr = (&(elf_ptr->PHDR(64))); + break; + case ARM32: + phdr = (&(elf_ptr->PHDR(32))); + break; + default: + exit(1); + + } + + /* + * Depending on whether the ISA is 32- or 64-bit determines the size of + * many of the fields in the Progra Header. This switch case deals with it. + */ + switch (elf_ptr->isa) { + + // 64-Bit ISAs + case X86_64: + case AARCH64: + /* + * p_type identifies what type of segment this is + * PT_NULL: 0x0 // Unused + * PT_LOAD: 0x1 // Loadable Segment + * PT_DYNAMIC: 0x2 // Dynamic Linker Information + * PT_INTERP: 0x3 // Interpreter Information + * PT_NOTE: 0x4 // Auxiliary Information + * PT_SHLIB: 0x5 // Reserved + * PT_PHDR: 0x6 // Segment with Program Header + * PT_TLS: 0x7 // Thread Local Storage + */ + REF_PHDR(64, p_type) = PT_LOAD; // 0x1 + + /* + * p_flags defines permissions for this section + * PF_R: 0x4 // Read + * PF_W: 0x2 // Write + * PF_X: 0x1 // Execute + */ + REF_PHDR(64, p_flags) = PF_R | PF_X; // 0x5 + + /* + * p_offset is the offset in the file image (relative to the start + * of the program header) for this segment. + */ + REF_PHDR(64, p_offset) = 0x0; + + /* + * p_vaddr is the virtual address where this segment should be loaded + * p_paddr is for the physical address (unused by System V) + */ + REF_PHDR(64, p_vaddr) = 0x400000; + REF_PHDR(64, p_paddr) = 0x400000; + + /* + * p_filesz is the size of the segment in the file image + * p_memsz is the size of the segment in memory + * + * Note: p_filesz doesn't have to equal p_memsz + */ + REF_PHDR(64, p_filesz) = elf_ptr->text.text_size; + REF_PHDR(64, p_memsz) = elf_ptr->text.text_size; + + break; + // 32-Bit ISAs + case ARM32: + /* + * p_type identifies what type of segment this is + * PT_NULL: 0x0 // Unused + * PT_LOAD: 0x1 // Loadable Segment + * PT_DYNAMIC: 0x2 // Dynamic Linker Information + * PT_INTERP: 0x3 // Interpreter Information + * PT_NOTE: 0x4 // Auxiliary Information + * PT_SHLIB: 0x5 // Reserved + * PT_PHDR: 0x6 // Segment with Program Header + * PT_TLS: 0x7 // Thread Local Storage + */ + REF_PHDR(32, p_type) = PT_LOAD; // 0x1 + + /* + * p_flags defines permissions for this section + * PF_R: 0x4 // Read + * PF_W: 0x2 // Write + * PF_X: 0x1 // Execute + */ + REF_PHDR(32, p_flags) = PF_R | PF_X; // 0x5 + + /* + * p_offset is the offset in the file image (relative to the start + * of the program header) for this segment. + */ + REF_PHDR(32, p_offset) = 0x0; + + /* + * p_vaddr is the virtual address where this segment should be loaded + * p_paddr is for the physical address (unused by System V) + */ + REF_PHDR(32, p_vaddr) = 0x10000; + REF_PHDR(32, p_paddr) = 0x10000; + + /* + * p_filesz is the size of the segment in the file image + * p_memsz is the size of the segment in memory + * + * Note: p_filesz doesn't have to equal p_memsz + */ + REF_PHDR(32, p_filesz) = elf_ptr->text.text_size; + REF_PHDR(32, p_memsz) = elf_ptr->text.text_size; + + break; + default: + exit(1); + + } + + /* + * p_align is the memory alignment + * + * Note: p_vaddr = p_offset % p_align + */ + switch (elf_ptr->isa) { + + case X86_64: + REF_PHDR(64, p_align) = 0x400000; + break; + case ARM32: + REF_PHDR(32, p_align) = 0x10000; + break; + case AARCH64: + REF_PHDR(64, p_align) = 0x400000; + break; + + } + + return phdr; + +} + +/* + * e_entry depends on p_vaddr, so has to be set after populate_ehdr() + * and populate_phdr() have been called. + * taken from libgolf.h + */ +int set_entry_point(RawBinary *elf_ptr) { + + /* + * Once the whole ELF file is copied into memory, control is handed to + * e_entry. Relative to the process's virtual memory address, the .text + * segment will be located immediately after the ELF and program header. + * + * ehdr and phdr are pointers to the ELF and Program headers respectively. + * The switch case casts and assigns them to the correct fields of the ELF + * struct, then sets ehdr->e_entry. + */ + void *ehdr, *phdr; + + switch (elf_ptr->isa) { + + case X86_64: + case AARCH64: + ehdr = GET_EHDR(64); + phdr = GET_PHDR(64); + REF_EHDR(64, e_entry) = REF_PHDR(64, p_vaddr) + ehdr_size + phdr_size; + break; + case ARM32: + ehdr = GET_EHDR(32); + phdr = GET_PHDR(32); + REF_EHDR(32, e_entry) = REF_PHDR(32, p_vaddr) + ehdr_size + phdr_size; + break; + default: + exit(1); + + } + + return 0; + +} + +typedef struct my_mutator { + + afl_state_t *afl; + size_t trim_size_current; + int trimmming_steps; + int cur_step; + u8 *mutated_out, *post_process_buf, *trim_buf; + +} my_mutator_t; + +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { + + srand(seed); // needed also by surgical_havoc_mutate() + my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + if ((data->mutated_out = (u8 *)malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init malloc"); + return NULL; + + } + + if ((data->post_process_buf = (u8 *)malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init malloc"); + return NULL; + + } + + if ((data->trim_buf = (u8 *)malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init malloc"); + return NULL; + + } + + data->afl = afl; + return data; + +} + +size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *in_buf, size_t buf_size, + u8 **out_buf, uint8_t *add_buf, + size_t add_buf_size, // add_buf can be NULL + size_t max_size) { + + RawBinary elf_obj; + RawBinary *elf = &elf_obj; + elf->isa = 62; + Elf64_Ehdr *ehdr; + Elf64_Phdr *phdr; + copy_text_segment(elf, buf, sizeof(buf)); + ehdr = populate_ehdr(elf); + phdr = populate_phdr(elf); + set_entry_point(elf); + + size_t mutated_size = ehdr_size + phdr_size + elf->text.text_size; + int pos = 0; + // example fields + ehdr->e_ident[EI_CLASS] = (uint8_t *)(in_buf + pos++); + ehdr->e_ident[EI_DATA] = (uint8_t *)(in_buf + pos++); + ehdr->e_ident[EI_VERSION] = (uint8_t *)(in_buf + pos++); + ehdr->e_ident[EI_OSABI] = (uint8_t *)(in_buf + pos++); + for (int i = 0x8; i < 0x10; ++i) { + + (ehdr->e_ident)[i] = (uint8_t *)(in_buf + pos++); + + } + + ehdr->e_version = (uint32_t *)(in_buf + pos); + pos += 4; + // sections headers + ehdr->e_shoff = (uint64_t *)(in_buf + pos); + pos += 8; + ehdr->e_shentsize = (uint16_t *)(in_buf + pos); + pos += 2; + ehdr->e_shnum = (uint16_t *)(in_buf + pos); + pos += 2; + ehdr->e_shstrndx = (uint16_t *)(in_buf + pos); + pos += 2; + ehdr->e_flags = (uint32_t *)(in_buf + pos); + pos += 4; + // physical addr + phdr->p_paddr = (uint64_t *)(in_buf + pos); + pos += 8; + phdr->p_align = (uint64_t *)(in_buf + pos); + pos += 8; + + /* mimic GEN_ELF() + * Write: + * - ELF Header + * - Program Header + * - Text Segment + */ + memcpy(data->mutated_out, ehdr, ehdr_size); + memcpy(data->mutated_out + ehdr_size, phdr, phdr_size); + memcpy(data->mutated_out + ehdr_size + phdr_size, elf->text.text_segment, + elf->text.text_size); + + *out_buf = data->mutated_out; + return mutated_size; + +} + +void afl_custom_deinit(my_mutator_t *data) { + + free(data->post_process_buf); + free(data->mutated_out); + free(data->trim_buf); + free(data); + +} + diff --git a/docs/Changelog.md b/docs/Changelog.md index 2c747e42..fa9099c0 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -3,6 +3,18 @@ This is the list of all noteworthy changes made in every public release of the tool. See README.md for the general instruction manual. +### Version ++4.09a (dev) + - afl-fuzz: + - added `AFL_FINAL_SYNC` which forces a final fuzzer sync (also for `-F`) + before terminating. + - afl-whatsup: + - now also shows coverage reached + - option -m shows only very relevant stats + - option -n will not use color in the output + - added benchmark/benchmark.sh if you want to see how good your fuzzing + speed is in comparison to other setups. + + ### Version ++4.08c (release) - afl-fuzz: - new mutation engine: mutations that favor discovery more paths are @@ -18,7 +30,6 @@ - -l X option to enable base64 transformation solving - allow to disable CMPLOG with '-c -' (e.g. afl.rs enforces '-c 0' on every instance which is counterproductive). - - afl-cmin/afl-cmin.bash: - fixed a bug inherited from vanilla AFL where a coverage of map[123] = 11 would be the same as map[1123] = 1 @@ -36,7 +47,6 @@ - qemu_mode: - added qemu_mode/utils/qemu_get_symbol_addr.sh - ### Version ++4.07c (release) - afl-fuzz: - reverse reading the seeds only on restarts (increases performance) @@ -65,7 +75,6 @@ - TritonDSE in custom_mutators/aflpp_tritondse - SymQEMU in custom_mutators/symqemu - ### Version ++4.06c (release) - afl-fuzz: - ensure temporary file descriptor is closed when not used diff --git a/docs/FAQ.md b/docs/FAQ.md index 9275eb94..242a379b 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -29,8 +29,8 @@ If you find an interesting or important question missing, submit it via which then implemented their own research and features, making it now by far the most flexible and feature rich guided fuzzer available as open source. And in independent fuzzing benchmarks it is one of the best fuzzers available, - e.g., [Fuzzbench - Report](https://www.fuzzbench.com/reports/2020-08-03/index.html). + e.g., + [Fuzzbench Report](https://www.fuzzbench.com/reports/2020-08-03/index.html). </p></details> <details> @@ -103,6 +103,42 @@ If you find an interesting or important question missing, submit it via to itself, this too would be an edge. </p></details> +<details> + <summary id="should-you-ever-stop-afl-fuzz-minimize-the-corpus-and-restart">Should you ever stop afl-fuzz, minimize the corpus and restart?</summary><p> + + To stop afl-fuzz, minimize it's corpus and restart you would usually do: + + ``` + Control-C # to terminate afl-fuzz + $ afl-cmin -T nproc -i out/default/queue -o minimized_queue -- ./target + $ AFL_FAST_CAL=1 AFL_CMPLOG_ONLY_NEW=1 afl-fuzz -i minimized_queue -o out2 [other options] -- ./target + ``` + + If this improves fuzzing or not is debated and no consensus has been reached + or in-depth analysis been performed. + + On the pro side: + * The queue/corpus is reduced (up to 20%) by removing intermediate paths + that are maybe not needed anymore. + + On the con side: + * Fuzzing time is lost for the time the fuzzing is stopped, minimized and + restarted. + + The the big question: + * Does a minimized queue/corpus improve finding new coverage or does it + hinder it? + + The AFL++ team's own limited analysis seem to to show that keeping + intermediate paths help to find more coverage, at least for afl-fuzz. + + For honggfuzz in comparison it is a good idea to restart it from time to + time if you have other fuzzers (e.g: AFL++) running in parallel to sync + the finds of other fuzzers to honggfuzz as it has no syncing feature like + AFL++ or libfuzzer. + +</p></details> + ## Targets <details> diff --git a/docs/env_variables.md b/docs/env_variables.md index affc9e3c..2ce274d3 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -412,10 +412,15 @@ checks or alter some of the more exotic semantics of the tool: set `AFL_IGNORE_PROBLEMS`. If you additionally want to also ignore coverage from late loaded libraries, you can set `AFL_IGNORE_PROBLEMS_COVERAGE`. - - When running in the `-M` or `-S` mode, setting `AFL_IMPORT_FIRST` causes the - fuzzer to import test cases from other instances before doing anything else. - This makes the "own finds" counter in the UI more accurate. Beyond counter - aesthetics, not much else should change. + - When running with multiple afl-fuzz or with `-F`, setting `AFL_IMPORT_FIRST` + causes the fuzzer to import test cases from other instances before doing + anything else. This makes the "own finds" counter in the UI more accurate. + + - When running with multiple afl-fuzz or with `-F`, setting `AFL_FINAL_SYNC` + will cause the fuzzer to perform a final import of test cases when + terminating. This is beneficial for `-M` main fuzzers to ensure it has all + unique test cases and hence you only need to `afl-cmin` this single + queue. - Setting `AFL_INPUT_LEN_MIN` and `AFL_INPUT_LEN_MAX` are an alternative to the afl-fuzz -g/-G command line option to control the minimum/maximum diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index ef84a18c..3dfd2b2c 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -158,6 +158,7 @@ struct queue_entry { u8 colorized, /* Do not run redqueen stage again */ cal_failed; /* Calibration failed? */ + bool trim_done, /* Trimmed? */ was_fuzzed, /* historical, but needed for MOpt */ passed_det, /* Deterministic stages passed? */ @@ -169,17 +170,15 @@ struct queue_entry { disabled; /* Is disabled from fuzz selection */ u32 bitmap_size, /* Number of bits set in bitmap */ - fuzz_level, /* Number of fuzzing iterations */ - n_fuzz_entry /* offset in n_fuzz */ #ifdef INTROSPECTION - , stats_selected, /* stats: how often selected */ stats_skipped, /* stats: how often skipped */ stats_finds, /* stats: # of saved finds */ stats_crashes, /* stats: # of saved crashes */ - stats_tmouts /* stats: # of saved timeouts */ + stats_tmouts, /* stats: # of saved timeouts */ #endif - ; + fuzz_level, /* Number of fuzzing iterations */ + n_fuzz_entry; /* offset in n_fuzz */ u64 exec_us, /* Execution time (us) */ handicap, /* Number of queue cycles behind */ @@ -402,7 +401,8 @@ typedef struct afl_env_vars { afl_exit_on_seed_issues, afl_try_affinity, afl_ignore_problems, afl_keep_timeouts, afl_no_crash_readme, afl_ignore_timeouts, afl_no_startup_calibration, afl_no_warn_instability, - afl_post_process_keep_original, afl_crashing_seeds_as_new_crash; + afl_post_process_keep_original, afl_crashing_seeds_as_new_crash, + afl_final_sync; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_preload, diff --git a/include/config.h b/include/config.h index 5a81c4e2..6a75737f 100644 --- a/include/config.h +++ b/include/config.h @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.08c" +#define VERSION "++4.09a" /****************************************************** * * diff --git a/include/envs.h b/include/envs.h index 0007d5a8..3f5a9e1c 100644 --- a/include/envs.h +++ b/include/envs.h @@ -59,6 +59,7 @@ static char *afl_environment_variables[] = { "AFL_EXIT_ON_TIME", "AFL_EXIT_ON_SEED_ISSUES", "AFL_FAST_CAL", + "AFL_FINAL_SYNC", "AFL_FORCE_UI", "AFL_FRIDA_DEBUG_MAPS", "AFL_FRIDA_DRIVER_NO_HOOK", diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 87157cad..0429db34 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -474,7 +474,7 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { /* Generating a hash on every input is super expensive. Bad idea and should only be used for special schedules */ - if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) { + if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) { classify_counts(&afl->fsrv); classified = 1; diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 48fd33ec..14ba1ace 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -701,13 +701,20 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { u64 fav_factor; u64 fuzz_p2; - if (unlikely(afl->schedule >= FAST && afl->schedule < RARE)) + if (likely(afl->schedule >= FAST && afl->schedule < RARE)) { + fuzz_p2 = 0; // Skip the fuzz_p2 comparison - else if (unlikely(afl->schedule == RARE)) + + } else if (unlikely(afl->schedule == RARE)) { + fuzz_p2 = next_pow2(afl->n_fuzz[q->n_fuzz_entry]); - else + + } else { + fuzz_p2 = q->fuzz_level; + } + if (unlikely(afl->schedule >= RARE) || unlikely(afl->fixed_seed)) { fav_factor = q->len << 2; @@ -729,47 +736,32 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { /* Faster-executing or smaller test cases are favored. */ u64 top_rated_fav_factor; u64 top_rated_fuzz_p2; - if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) - top_rated_fuzz_p2 = - next_pow2(afl->n_fuzz[afl->top_rated[i]->n_fuzz_entry]); - else - top_rated_fuzz_p2 = afl->top_rated[i]->fuzz_level; - if (unlikely(afl->schedule >= RARE) || unlikely(afl->fixed_seed)) { + if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) { - top_rated_fav_factor = afl->top_rated[i]->len << 2; + top_rated_fuzz_p2 = + next_pow2(afl->n_fuzz[afl->top_rated[i]->n_fuzz_entry]); } else { - top_rated_fav_factor = - afl->top_rated[i]->exec_us * afl->top_rated[i]->len; - - } - - if (fuzz_p2 > top_rated_fuzz_p2) { - - continue; - - } else if (fuzz_p2 == top_rated_fuzz_p2) { - - if (fav_factor > top_rated_fav_factor) { continue; } + top_rated_fuzz_p2 = afl->top_rated[i]->fuzz_level; } if (unlikely(afl->schedule >= RARE) || unlikely(afl->fixed_seed)) { - if (fav_factor > afl->top_rated[i]->len << 2) { continue; } + top_rated_fav_factor = afl->top_rated[i]->len << 2; } else { - if (fav_factor > - afl->top_rated[i]->exec_us * afl->top_rated[i]->len) { + top_rated_fav_factor = + afl->top_rated[i]->exec_us * afl->top_rated[i]->len; - continue; + } - } + if (likely(fuzz_p2 > top_rated_fuzz_p2)) { continue; } - } + if (likely(fav_factor > top_rated_fav_factor)) { continue; } /* Looks like we're going to win. Decrease ref count for the previous winner, discard its afl->fsrv.trace_bits[] if necessary. */ diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 5a6b95cf..97e00415 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -269,6 +269,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_import_first = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_FINAL_SYNC", + + afl_environment_variable_len)) { + + afl->afl_env.afl_final_sync = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_CUSTOM_MUTATOR_ONLY", afl_environment_variable_len)) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index cdb3f996..43834172 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -299,6 +299,7 @@ static void usage(u8 *argv0, int more_help) { "AFL_NO_FORKSRV: run target via execve instead of using the forkserver\n" "AFL_NO_SNAPSHOT: do not use the snapshot feature (if the snapshot lkm is loaded)\n" "AFL_NO_STARTUP_CALIBRATION: no initial seed calibration, start fuzzing at once\n" + "AFL_NO_WARN_INSTABILITY: no warn about instability issues on startup calibration\n" "AFL_NO_UI: switch status screen off\n" "AFL_NYX_AUX_SIZE: size of the Nyx auxiliary buffer. Must be a multiple of 4096.\n" " Increase this value in case the crash reports are truncated.\n" @@ -2899,6 +2900,16 @@ stop_fuzzing: time_spent_working / afl->fsrv.total_execs); #endif + if (afl->afl_env.afl_final_sync) { + + SAYF(cYEL "[!] " cRST + "\nPerforming final sync, this make take some time ...\n"); + sync_fuzzers(afl); + write_bitmap(afl); + SAYF(cYEL "[!] " cRST "Done!\n\n"); + + } + if (afl->is_main_node) { u8 path[PATH_MAX]; |