diff options
| author | van Hauser <vh@thc.org> | 2023-02-25 09:20:47 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-02-25 09:20:47 +0100 |
| commit | ea14f3fd40e32234989043a525e3853fcb33c1b6 (patch) | |
| tree | aaa8770a743f6ab25a9f2849f62af8a68521344d | |
| parent | 3881ccd0b7520f67fd0b34f010443dc249cbc8f1 (diff) | |
| parent | 2b5c9954f6099d7a70cbcf2db9af5a19327f385f (diff) | |
| download | afl++-ea14f3fd40e32234989043a525e3853fcb33c1b6.tar.gz | |
Merge pull request #1653 from AFLplusplus/dev
push to stable
43 files changed, 1880 insertions, 253 deletions
diff --git a/.gitignore b/.gitignore index 45d8676c..c01750e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,104 +1,107 @@ -.test -.test2 -.sync_tmp -.vscode +!coresight_mode +!coresight_mode/coresight-trace +*.dSYM *.o +*.o.tmp +*.pyc *.so *.swp -*.pyc -*.dSYM -as -a.out -ld -in -out -core* -compile_commands.json +.sync_tmp +.test +.test2 +.vscode afl-analyze +afl-analyze.8 afl-as +afl-as.8 +afl-c++ +afl-c++.8 +afl-cc +afl-cc.8 afl-clang afl-clang++ afl-clang-fast afl-clang-fast++ +afl-clang-fast++.8 +afl-clang-fast.8 afl-clang-lto afl-clang-lto++ +afl-clang-lto++.8 +afl-clang-lto.8 +afl-cmin.8 +afl-cmin.bash.8 +afl-cs-proxy +afl-frida-trace.so afl-fuzz +afl-fuzz.8 afl-g++ +afl-g++.8 afl-gcc +afl-gcc.8 afl-gcc-fast +afl-gcc-fast.8 afl-g++-fast +afl-g++-fast.8 afl-gotcpu +afl-gotcpu.8 afl-ld afl-ld-lto -afl-cs-proxy +afl-lto +afl-lto++ +afl-lto++.8 +afl-lto.8 +afl-persistent-config.8 +afl-plot.8 afl-qemu-trace afl-showmap -afl-tmin -afl-analyze.8 -afl-as.8 -afl-clang-fast++.8 -afl-clang-fast.8 -afl-clang-lto.8 -afl-clang-lto++.8 -afl-cmin.8 -afl-cmin.bash.8 -afl-fuzz.8 -afl-c++.8 -afl-cc.8 -afl-gcc.8 -afl-g++.8 -afl-gcc-fast.8 -afl-g++-fast.8 -afl-gotcpu.8 -afl-plot.8 afl-showmap.8 afl-system-config.8 +afl-tmin afl-tmin.8 afl-whatsup.8 -afl-persistent-config.8 -afl-c++ -afl-cc -afl-lto -afl-lto++ -afl-lto++.8 -afl-lto.8 -qemu_mode/libcompcov/compcovtest -qemu_mode/qemu-* -qemu_mode/qemuafl -unicorn_mode/samples/*/\.test-* -unicorn_mode/samples/*/output/ -test/unittests/unit_maybe_alloc -test/unittests/unit_preallocable -test/unittests/unit_list -test/unittests/unit_rand -test/unittests/unit_hash -examples/afl_network_proxy/afl-network-server -examples/afl_network_proxy/afl-network-client +a.out +as +compile_commands.json +core* examples/afl_frida/afl-frida -examples/afl_frida/libtestinstr.so examples/afl_frida/frida-gum-example.c examples/afl_frida/frida-gum.h +examples/afl_frida/libtestinstr.so +examples/afl_network_proxy/afl-network-client +examples/afl_network_proxy/afl-network-server examples/aflpp_driver/libAFLDriver.a examples/aflpp_driver/libAFLQemuDriver.a +gmon.out +in +ld libAFLDriver.a libAFLQemuDriver.a +out +qemu_mode/libcompcov/compcovtest +qemu_mode/qemu-* +qemu_mode/qemuafl test/.afl_performance test-instr test/output +test/test-c +test/test-cmplog +test/test-compcov test/test-instr.ts test/test-persistent -gmon.out -afl-frida-trace.so +test/unittests/unit_hash +test/unittests/unit_list +test/unittests/unit_maybe_alloc +test/unittests/unit_preallocable +test/unittests/unit_rand +unicorn_mode/samples/*/output/ +unicorn_mode/samples/*/\.test-* utils/afl_network_proxy/afl-network-client utils/afl_network_proxy/afl-network-server -utils/plot_ui/afl-plot-ui -*.o.tmp utils/afl_proxy/afl-proxy utils/optimin/build utils/optimin/optimin utils/persistent_mode/persistent_demo utils/persistent_mode/persistent_demo_new utils/persistent_mode/test-instr -!coresight_mode -!coresight_mode/coresight-trace -vuln_prog \ No newline at end of file +utils/plot_ui/afl-plot-ui +vuln_prog diff --git a/Dockerfile b/Dockerfile index fd47a59f..59ce8778 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,7 @@ RUN apt-get update && \ clang-${LLVM_VERSION} clang-tools-${LLVM_VERSION} libc++1-${LLVM_VERSION} \ libc++-${LLVM_VERSION}-dev libc++abi1-${LLVM_VERSION} libc++abi-${LLVM_VERSION}-dev \ libclang1-${LLVM_VERSION} libclang-${LLVM_VERSION}-dev \ - libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} \ + libclang-common-${LLVM_VERSION}-dev libclang-rt-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} \ libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \ liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev \ libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev libomp5-${LLVM_VERSION} \ diff --git a/README.md b/README.md index eeab7aa1..821b8cb7 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,7 @@ Thank you! (For people sending pull requests - please add yourself to this list Thomas Rooijakkers David Carlier Ruben ten Hove Joey Jiao fuzzah @intrigus-lgtm + Yaakov Saxon ``` </details> diff --git a/TODO.md b/TODO.md index 862224f0..187fa191 100644 --- a/TODO.md +++ b/TODO.md @@ -9,13 +9,12 @@ - afl-plot to support multiple plot_data - parallel builds for source-only targets - get rid of check_binary, replace with more forkserver communication - - first fuzzer should be a main automatically + - first fuzzer should be a main automatically? not sure. ## Maybe - forkserver tells afl-fuzz if cmplog is supported and if so enable it by default, with AFL_CMPLOG_NO=1 (?) set to skip? - - afl_custom_fuzz_splice_optin() - afl_custom_splice() - cmdline option from-to range for mutations diff --git a/custom_mutators/README.md b/custom_mutators/README.md index 0289e150..8d01856f 100644 --- a/custom_mutators/README.md +++ b/custom_mutators/README.md @@ -11,6 +11,16 @@ The `./examples` folder contains examples for custom mutators in python and C. In `./rust`, you will find rust bindings, including a simple example in `./rust/example` and an example for structured fuzzing, based on lain, in`./rust/example_lain`. +## The AFL++ grammar agnostic grammar mutator + +In `./autotokens` you find a token-level fuzzer that does not need to know +anything about the grammar of an input as long as it is in ascii and allows +whitespace. +It is very fast and effective. + +If you are looking for an example of how to effectively create a custom +mutator take a look at this one. + ## The AFL++ Grammar Mutator If you use git to clone AFL++, then the following will incorporate our diff --git a/custom_mutators/autotokens/Makefile b/custom_mutators/autotokens/Makefile new file mode 100644 index 00000000..0daba17d --- /dev/null +++ b/custom_mutators/autotokens/Makefile @@ -0,0 +1,26 @@ +ifdef debug + CPPLAGS += -fsanitize=address + CXXFLAGS += -Wall + CC := clang + CXX := clang++ +endif +ifdef DEBUG + CPPFLAGS += -fsanitize=address + CXXFLAGS += -Wall + CC := clang + CXX := clang++ +endif + +all: autotokens.so + +afl-fuzz-queue.o: ../../src/afl-fuzz-queue.c + $(CC) -D_STANDALONE_MODULE=1 -I../../include -g -O3 $(CPPFLAGS) -fPIC -c -o ./afl-fuzz-queue.o ../../src/afl-fuzz-queue.c + +afl-common.o: ../../src/afl-common.c + $(CC) -I../../include -g -O3 $(CPPFLAGS) -DBIN_PATH=\"dummy\" -Wno-pointer-sign -fPIC -c -o ./afl-common.o ../../src/afl-common.c + +autotokens.so: afl-fuzz-queue.o afl-common.o autotokens.cpp + $(CXX) -Wno-deprecated -g -O3 $(CXXFLAGS) $(CPPFLAGS) -shared -fPIC -o autotokens.so -I../../include autotokens.cpp ./afl-fuzz-queue.o ../../src/afl-performance.o ./afl-common.o + +clean: + rm -f autotokens.so *.o *~ core diff --git a/custom_mutators/autotokens/README b/custom_mutators/autotokens/README new file mode 100644 index 00000000..cca168fd --- /dev/null +++ b/custom_mutators/autotokens/README @@ -0,0 +1,34 @@ +# Autotokens + +This implements an improved autotoken grammar fuzzing idea presented in +[Token-Level Fuzzing][https://www.usenix.org/system/files/sec21-salls.pdf]. +It is a grammar fuzzer without actually knowing the grammar, but only works +with text based inputs. + +It is recommended to run with together in an instance with `CMPLOG`. + +If you have a dictionary (`-x`) this improves this custom grammar mutator. + +If **not** running with `CMPLOG`, it is possible to set +`AFL_CUSTOM_MUTATOR_ONLY` to concentrate on grammar bug classes. + +Do **not** set `AFL_DISABLE_TRIM` with this custom mutator! + +## Configuration via environment variables + +`AUTOTOKENS_ONLY_FAV` - only use this mutator on favorite queue items +`AUTOTOKENS_COMMENT` - what character or string starts a comment which will be + removed. Default: `/* ... */` +`AUTOTOKENS_FUZZ_COUNT_SHIFT` - reduce the number of fuzzing performed, shifting + the value by this number, e.g. 1. +`AUTOTOKENS_AUTO_DISABLE` - disable this module if the seeds are not ascii + (or no input and no (ascii) dictionary) +`AUTOTOKENS_LEARN_DICT` - learn from dictionaries? + 0 = none + 1 = only -x or autodict + 2 = -x, autodict and `CMPLOG` +`AUTOTOKENS_CHANGE_MIN` - minimum number of mutations (1-256, default 8) +`AUTOTOKENS_CHANGE_MAX` - maximum number of mutations (1-4096, default 64) +`AUTOTOKENS_CREATE_FROM_THIN_AIR` - if only one small start file is present and + a dictionary loaded then create one initial + structure based on the dictionary. diff --git a/custom_mutators/autotokens/autotokens.cpp b/custom_mutators/autotokens/autotokens.cpp new file mode 100644 index 00000000..8135aba1 --- /dev/null +++ b/custom_mutators/autotokens/autotokens.cpp @@ -0,0 +1,1101 @@ +/* + token level fuzzing custom mutator for afl++ + (c) by Marc Heuse <mh@mh-sec.de> + License: Apache 2.0 +*/ + +extern "C" { + +#include "afl-fuzz.h" + +} + +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> + +#include <iostream> +#include <fstream> +#include <unordered_map> +#include <vector> +#include <regex> + +#define AUTOTOKENS_DEBUG 0 +#define AUTOTOKENS_ONLY_FAV 0 +#define AUTOTOKENS_CHANGE_MIN 8 +#define AUTOTOKENS_CHANGE_MAX 64 +#define AUTOTOKENS_SIZE_MIN 8 +#define AUTOTOKENS_SIZE_MAX 65535 +#define AUTOTOKENS_SPLICE_MIN 4 +#define AUTOTOKENS_SPLICE_MAX 64 +#define AUTOTOKENS_CREATE_FROM_THIN_AIR 0 +#define AUTOTOKENS_FUZZ_COUNT_SHIFT 0 +#define AUTOTOKENS_AUTO_DISABLE 0 +// 0 = no learning, 1 only from -x dict/autodict, 2 also from cmplog +#define AUTOTOKENS_LEARN_DICT 1 +#ifndef AUTOTOKENS_SPLICE_DISABLE + #define AUTOTOKENS_SPLICE_DISABLE 0 +#endif +#ifndef AFL_TXT_MAX_LEN + #define AFL_TXT_MAX_LEN 65535 +#endif + +#if AUTOTOKENS_SPLICE_MIN >= AUTOTOKENS_SIZE_MIN + #error SPLICE_MIN must be lower than SIZE_MIN +#endif + +using namespace std; + +typedef struct my_mutator { + + afl_state *afl; + +} my_mutator_t; + +#undef DEBUGF +#define DEBUGF \ + if (unlikely(debug)) fprintf +#define IFDEBUG if (unlikely(debug)) + +static afl_state *afl_ptr; +static int module_disabled = 0; +static int auto_disable = AUTOTOKENS_AUTO_DISABLE; +static int debug = AUTOTOKENS_DEBUG; +static int only_fav = AUTOTOKENS_ONLY_FAV; +static int learn_dictionary_tokens = AUTOTOKENS_LEARN_DICT; +static int fuzz_count_shift = AUTOTOKENS_FUZZ_COUNT_SHIFT; +static int create_from_thin_air = AUTOTOKENS_CREATE_FROM_THIN_AIR; +static int change_min = AUTOTOKENS_CHANGE_MIN; +static int change_max = AUTOTOKENS_CHANGE_MAX; +static u32 current_id; +static u32 valid_structures; +static u32 whitespace_ids; +static u32 extras_cnt, a_extras_cnt; +static u64 all_spaces, all_tabs, all_lf, all_ws; +static u64 all_structure_items; +static u64 fuzz_count; +static unordered_map<string, vector<u32> *> file_mapping; +static unordered_map<u32, vector<u32> *> id_mapping; +static unordered_map<string, u32> token_to_id; +static unordered_map<u32, string> id_to_token; +static string output; +static regex *regex_comment_custom; +// multiline requires g++-11 libs :( +static regex regex_comment_star( + "/\\*([:print:]|\n)*?\\*/", + regex_constants::optimize /* | regex_constants::multiline */); +static regex regex_word("[A-Za-z0-9_$.-]+", regex::optimize); +static regex regex_whitespace(R"([ \t]+)", regex::optimize); +static vector<u32> *s; // the structure of the currently selected input + +// FUNCTIONS + +/* This function is called once after everything is set up but before + any fuzzing attempt has been performed. + This is called in afl_custom_queue_get() */ +static void first_run(void *data) { + + (void)(data); + + /* For auto-loading this module we check here if we can analyze from the + input if the inputs look like text inputs and disable the module if + not. */ + + if (afl_ptr->custom_only || !auto_disable) { return; } + + if (unlikely(afl_ptr->active_items == 1 && + afl_ptr->queue_cur->len < AFL_TXT_MIN_LEN)) { + + if (afl_ptr->extras_cnt > 8) { + + u32 valid = 0; + + while (extras_cnt < afl_ptr->extras_cnt) { + + u32 ok = 1, l = afl_ptr->extras[extras_cnt].len; + u8 *buf, *ptr = afl_ptr->extras[extras_cnt].data; + + for (u32 i = 0; i < l; ++i) { + + if (!isascii((int)ptr[i]) && !isprint((int)ptr[i])) { + + ok = 0; + break; + + } + + } + + if (ok) { + + buf = (u8 *)malloc(afl_ptr->extras[extras_cnt].len + 1); + memcpy(buf, afl_ptr->extras[extras_cnt].data, + afl_ptr->extras[extras_cnt].len); + buf[afl_ptr->extras[extras_cnt].len] = 0; + token_to_id[(char *)buf] = current_id; + id_to_token[current_id] = (char *)buf; + ++current_id; + ++valid; + + } + + ++extras_cnt; + + } + + if ((valid * 100) / afl_ptr->extras_cnt <= 70) { module_disabled = 1; } + + DEBUGF(stderr, "DICT: total %u, valid %u, %u <= 70 == disable\n", + afl_ptr->extras_cnt, valid, + (u32)((valid * 100) / afl_ptr->extras_cnt)); + + } else { + + module_disabled = 1; + + } + + return; + + } + + u32 is_ascii = 0, valid = 0; + + for (u32 i = 0; i < afl_ptr->queued_items; ++i) { + + struct queue_entry *q; + + q = afl_ptr->queue_buf[i]; + + if (!q->disabled && q->len >= AUTOTOKENS_SIZE_MIN && + q->len <= AFL_TXT_MAX_LEN) { + + ++valid; + u8 *input = queue_testcase_get(afl_ptr, q); + + u32 valid_chars = 0; + for (u32 i = 0; i < q->len; ++i) { + + if (isascii((int)input[i]) || isprint((int)input[i])) { ++valid_chars; } + + } + + // we want at least 99% of text characters ... + if (((q->len * AFL_TXT_MIN_PERCENT) / 100) <= valid_chars) { + + ++is_ascii; + q->is_ascii = 1; + + } + + } + + } + + if ((is_ascii * 100) / valid <= 70) { module_disabled = 1; } + + DEBUGF(stderr, "seeds: total %u, valid %u, ascii %u, %u <= 70 == disabled\n", + afl_ptr->active_items, valid, is_ascii, + (u32)((is_ascii * 100) / valid)); + +} + +static u32 good_whitespace_or_singleval() { + + u32 i = rand_below(afl_ptr, current_id); + if (id_to_token[i].size() == 1) { return i; } + i = rand_below(afl_ptr, all_ws); + if (i < all_spaces) { + + return 0; + + } else if (i < all_tabs) { + + return 1; + + } else + + return 2; // linefeed + +} + +extern "C" u32 afl_custom_fuzz_count(void *data, const u8 *buf, + size_t buf_size) { + + (void)(data); + + if (s == NULL) return 0; + + u32 shift = unlikely(afl_ptr->custom_only) ? 7 : 8; + u32 stage_max = (u32)((HAVOC_CYCLES * afl_ptr->queue_cur->perf_score) / + afl_ptr->havoc_div) >> + shift; + if (fuzz_count_shift) { stage_max >>= (u32)fuzz_count_shift; }; + DEBUGF(stderr, "fuzz count: %u\n", stage_max); + + return stage_max; + +} + +extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, + u8 **out_buf, u8 *add_buf, + size_t add_buf_size, size_t max_size) { + + (void)(data); + + if (unlikely(s == NULL)) { + + *out_buf = NULL; + return 0; + + } + + vector<u32> m = *s; // copy of the structure we will modify + u32 i, m_size = (u32)m.size(); + + u32 rounds = + MIN(change_max, + MAX(change_min, + MIN(m_size >> 3, HAVOC_CYCLES * afl_ptr->queue_cur->perf_score * + afl_ptr->havoc_div / 256))); + // DEBUGF(stderr, "structure size: %lu, rounds: %u \n", m.size(), rounds); + +#if AUTOTOKENS_SPLICE_DISABLE == 1 + #define AUTOTOKENS_MUT_MAX 18 +#else + #define AUTOTOKENS_MUT_MAX 27 +#endif + + u32 max_rand = AUTOTOKENS_MUT_MAX, new_item, pos; + + for (i = 0; i < rounds; ++i) { + + switch (rand_below(afl_ptr, max_rand)) { + + /* CHANGE/MUTATE single item */ + case 0 ... 9: { + + pos = rand_below(afl_ptr, m_size); + u32 cur_item = m[pos]; + do { + + new_item = rand_below(afl_ptr, current_id); + + } while (unlikely( + + new_item == cur_item || + ((whitespace_ids < new_item && whitespace_ids >= cur_item) || + (whitespace_ids >= new_item && whitespace_ids < cur_item)))); + + // DEBUGF(stderr, "MUT: %u -> %u\n", cur_item, new_item); + m[pos] = new_item; + break; + + } + + /* INSERT (m_size +1 so we insert also after last place) */ + case 10 ... 13: { + + do { + + new_item = rand_below(afl_ptr, current_id); + + } while (unlikely(new_item >= whitespace_ids)); + + u32 pos = rand_below(afl_ptr, m_size + 1); + m.insert(m.begin() + pos, new_item); + ++m_size; + // DEBUGF(stderr, "INS: %u at %u\n", new_item, pos); + + break; + + } + +#if AUTOTOKENS_SPLICE_DISABLE != 1 + /* SPLICING */ + case 14 ... 22: { + + u32 strategy = rand_below(afl_ptr, 4), dst_off, n; + auto src = id_mapping[rand_below(afl_ptr, valid_structures)]; + u32 src_size = src->size(); + u32 src_off = rand_below(afl_ptr, src_size - AUTOTOKENS_SPLICE_MIN); + u32 rand_r = 1 + MAX(AUTOTOKENS_SPLICE_MIN, + MIN(AUTOTOKENS_SPLICE_MAX, src_size - src_off)); + + switch (strategy) { + + // insert + case 0: { + + dst_off = rand_below(afl_ptr, m_size); + n = AUTOTOKENS_SPLICE_MIN + + rand_below(afl_ptr, MIN(AUTOTOKENS_SPLICE_MAX, + rand_r - AUTOTOKENS_SPLICE_MIN)); + m.insert(m.begin() + dst_off, src->begin() + src_off, + src->begin() + src_off + n); + m_size += n; + // DEBUGF(stderr, "SPLICE-INS: %u at %u\n", n, dst_off); + + break; + + } + + // overwrite + default: { + + dst_off = rand_below(afl_ptr, m_size - AUTOTOKENS_SPLICE_MIN); + n = AUTOTOKENS_SPLICE_MIN + + rand_below( + afl_ptr, + MIN(AUTOTOKENS_SPLICE_MAX - AUTOTOKENS_SPLICE_MIN, + MIN(m_size - dst_off - AUTOTOKENS_SPLICE_MIN, + src_size - src_off - AUTOTOKENS_SPLICE_MIN))); + + copy(src->begin() + src_off, src->begin() + src_off + n, + m.begin() + dst_off); + + // DEBUGF(stderr, "SPLICE-MUT: %u at %u\n", n, dst_off); + break; + + } + + } + + break; + + } + +#endif + + /* ERASE - only if large enough */ + default: { + + if (m_size > 8) { + + do { + + pos = rand_below(afl_ptr, m_size); + + } while (unlikely(m[pos] < whitespace_ids)); + + m.erase(m.begin() + pos); + --m_size; + + } else { + + // if the data is already too small do not try to make it smaller + // again this run. + + max_rand -= 4; + + } + + break; + + } + + } + + } + + /* Now we create the output */ + + output = ""; + u32 prev_size = 1, was_whitespace = 1; + + for (i = 0; i < m_size; ++i) { + + if (likely(i + 1 < m_size)) { + + u32 this_size = id_to_token[m[i]].size(); + u32 is_whitespace = m[i] < whitespace_ids; + + /* The output we are generating might need repairing. + General rule: two items that have a size larger than 2 are strings + or identifizers and need a whitespace or an item of length 1 in + between. */ + if (unlikely(!(prev_size == 1 || was_whitespace || this_size == 1 || + is_whitespace))) { + + output += id_to_token[good_whitespace_or_singleval()]; + + } + + prev_size = this_size; + was_whitespace = is_whitespace; + + } + + output += id_to_token[m[i]]; + + } + + u32 mutated_size = (u32)output.size(); + u8 *mutated_out = (u8 *)output.data(); + + if (unlikely(mutated_size > max_size)) { mutated_size = max_size; } + + /* + IFDEBUG { + + DEBUGF(stderr, "MUTATED to %u bytes:\n", mutated_size); + fwrite(output.data(), 1, mutated_size, stderr); + DEBUGF(stderr, "\n---\n"); + + } + + */ + + *out_buf = mutated_out; + ++fuzz_count; + return mutated_size; + +} + +/* I get f*cking stack overflow using C++ regex with a regex of + "\"[[:print:]]*?\"" if this matches a long string even with regex::optimize + enabled :-( */ +static u8 my_search_string(string::const_iterator cur, + string::const_iterator ende, + string::const_iterator *match_begin, + string::const_iterator *match_end) { + + string::const_iterator start = cur, found_begin; + u8 quote_type = 0; + + while (cur < ende) { + + switch (*cur) { + + case '"': { + + if (cur == start || *(cur - 1) != '\\') { + + if (!quote_type) { + + found_begin = cur; + quote_type = 1; + + } else if (quote_type == 1) { + + *match_begin = found_begin; + *match_end = cur + 1; + return 1; + + } + + } + + break; + + } + + case '\'': { + + if (cur == start || *(cur - 1) != '\\') { + + if (!quote_type) { + + found_begin = cur; + quote_type = 2; + + } else if (quote_type == 2) { + + *match_begin = found_begin; + *match_end = cur + 1; + return 1; + + } + + } + + break; + + } + + case '\n': + case '\r': + case 0: { + + quote_type = 0; + break; + + } + + default: + if (unlikely(quote_type && !isprint(*cur))) { quote_type = 0; } + break; + + } + + ++cur; + + } + + return 0; + +} + +/* We are not using afl_custom_queue_new_entry() because not every corpus entry + will be necessarily fuzzed with this custom mutator. + So we use afl_custom_queue_get() instead. */ + +extern "C" unsigned char afl_custom_queue_get(void *data, + const unsigned char *filename) { + + static int learn_state = 0; + static int is_first_run = 1; + (void)(data); + + if (unlikely(is_first_run)) { + + is_first_run = 0; + first_run(data); + + if (module_disabled) { + + WARNF("Autotokens custom module is disabled."); + + } else if (auto_disable) { + + OKF("Autotokens custom module is enabled."); + + } + + } + + if (likely(module_disabled) || + (unlikely(!afl_ptr->custom_only) && !create_from_thin_air && + ((afl_ptr->shm.cmplog_mode && !afl_ptr->queue_cur->is_ascii) || + (only_fav && !afl_ptr->queue_cur->favored)))) { + + s = NULL; + DEBUGF(stderr, + "cmplog not ascii or only_fav and not favorite or disabled\n"); + return 1; + + } + + // check if there are new dictionary entries and add them to the tokens + if (unlikely(learn_state < learn_dictionary_tokens) && + likely(valid_structures || create_from_thin_air)) { + + if (unlikely(!learn_state)) { learn_state = 1; } + + while (extras_cnt < afl_ptr->extras_cnt) { + + u32 ok = 1, l = afl_ptr->extras[extras_cnt].len; + u8 *buf, *ptr = afl_ptr->extras[extras_cnt].data; + + for (u32 i = 0; i < l; ++i) { + + if (!isascii((int)ptr[i]) && !isprint((int)ptr[i])) { + + ok = 0; + break; + + } + + } + + if (ok) { + + buf = (u8 *)malloc(afl_ptr->extras[extras_cnt].len + 1); + memcpy(buf, afl_ptr->extras[extras_cnt].data, + afl_ptr->extras[extras_cnt].len); + buf[afl_ptr->extras[extras_cnt].len] = 0; + token_to_id[(char *)buf] = current_id; + id_to_token[current_id] = (char *)buf; + ++current_id; + + } + + ++extras_cnt; + + } + + while (a_extras_cnt < afl_ptr->a_extras_cnt) { + + u32 ok = 1, l = afl_ptr->a_extras[a_extras_cnt].len; + u8 *ptr = afl_ptr->a_extras[a_extras_cnt].data; + + for (u32 i = 0; i < l; ++i) { + + if (!isascii((int)ptr[i]) && !isprint((int)ptr[i])) { + + ok = 0; + break; + + } + + } + + if (ok) { + + token_to_id[(char *)ptr] = current_id; + id_to_token[current_id] = (char *)ptr; + ++current_id; + + } + + ++a_extras_cnt; + + } + + } + + vector<u32> *structure = NULL; + string fn = (char *)filename; + auto entry = file_mapping.find(fn); + + // if there is only one active queue item at start and it is very small + // the we create once a structure randomly. + if (unlikely(create_from_thin_air)) { + + if (current_id > whitespace_ids + 6 && afl_ptr->active_items == 1 && + afl_ptr->queue_cur->len < AFL_TXT_MIN_LEN) { + + DEBUGF(stderr, "Creating an entry from thin air...\n"); + structure = new vector<u32>(); + u32 item, prev, cnt = current_id >> 1; + structure->reserve(cnt + 4); + for (u32 i = 0; i < cnt; i++) { + + item = rand_below(afl_ptr, current_id); + if (i && id_to_token[item].length() > 1 && + id_to_token[prev].length() > 1) { + + structure->push_back(good_whitespace_or_singleval()); + + } + + structure->push_back(item); + prev = item; + + } + + s = structure; + file_mapping[fn] = structure; + id_mapping[valid_structures] = structure; + ++valid_structures; + all_structure_items += structure->size(); + + return 1; + + } + + create_from_thin_air = 0; + + } + + if (entry == file_mapping.end()) { + + // this input file was not analyzed for tokens yet, so let's do it! + size_t len = afl_ptr->queue_cur->len; + + if (len < AFL_TXT_MIN_LEN) { + + file_mapping[fn] = structure; // NULL ptr so we don't read the file again + s = NULL; + DEBUGF(stderr, "Too short (%lu) %s\n", len, filename); + return 1; + + } else if (len > AFL_TXT_MAX_LEN) { + + file_mapping[fn] = structure; // NULL ptr so we don't read the file again + s = NULL; + DEBUGF(stderr, "Too long (%lu) %s\n", len, filename); + return 1; + + } + + u8 *input_buf = queue_testcase_get(afl_ptr, afl_ptr->queue_cur); + string input((char *)input_buf, afl_ptr->queue_cur->len); + + if (!afl_ptr->shm.cmplog_mode) { + + // not running with CMPLOG? bad choice, but whatever ... + // we only want text inputs, so we have to check it ourselves. + + u32 valid_chars = 0; + for (u32 i = 0; i < len; ++i) { + + if (isascii((int)input[i]) || isprint((int)input[i])) { ++valid_chars; } + + } + + // we want at least 95% of text characters ... + if (((len * AFL_TXT_MIN_PERCENT) / 100) > valid_chars) { + + file_mapping[fn] = NULL; + s = NULL; + DEBUGF(stderr, "Not text (%lu) %s\n", len, filename); + return 1; + + } + + } + + // DEBUGF(stderr, "Read %lu bytes for %s\nBefore comment trim:\n%s\n", + // input.size(), filename, input.c_str()); + + if (regex_comment_custom) { + + input = regex_replace(input, *regex_comment_custom, "$2"); + + } else { + + input = regex_replace(input, regex_comment_star, ""); + + } + + DEBUGF(stderr, "After replace %lu bytes for %s\n%s\n", input.size(), + filename, input.c_str()); + + u32 spaces = count(input.begin(), input.end(), ' '); + u32 tabs = count(input.begin(), input.end(), '\t'); + u32 linefeeds = count(input.begin(), input.end(), '\n'); + bool ends_with_linefeed = input[input.length() - 1] == '\n'; + + DEBUGF(stderr, "spaces=%u tabs=%u linefeeds=%u ends=%u\n", spaces, tabs, + linefeeds, ends_with_linefeed); + + all_spaces += spaces; + all_tabs += tabs; + all_lf += linefeeds; + all_ws = all_spaces + all_tabs + all_lf; + + // now extract all tokens + vector<string> tokens; + string::const_iterator cur = input.begin(), ende = input.end(), found, prev, + match_begin, match_end; + + DEBUGF(stderr, "START!\n"); + + while (my_search_string(cur, ende, &match_begin, &match_end)) { + + prev = cur; + found = match_begin; + cur = match_end; + + IFDEBUG { + + string foo(match_begin, match_end); + DEBUGF(stderr, + "string %s found at start %lu offset %lu continue at %lu\n", + foo.c_str(), prev - input.begin(), found - prev, + cur - input.begin()); + + } + + if (prev < found) { // there are items between search start and find + while (prev < found) { + + if (isspace(*prev)) { + + auto start = prev; + while (isspace(*prev)) { + + ++prev; + + } + + tokens.push_back(std::string(start, prev)); + DEBUGF(stderr, "WHITESPACE %ld \"%s\"\n", prev - start, + tokens[tokens.size() - 1].c_str()); + + } else if (isalnum(*prev) || *prev == '$' || *prev == '_') { + + auto start = prev; + while (isalnum(*prev) || *prev == '$' || *prev == '_' || + *prev == '.' || *prev == '/') { + + ++prev; + + } + + tokens.push_back(string(start, prev)); + DEBUGF(stderr, "IDENTIFIER %ld \"%s\"\n", prev - start, + tokens[tokens.size() - 1].c_str()); + + } else { + + tokens.push_back(string(prev, prev + 1)); + DEBUGF(stderr, "OTHER \"%c\"\n", *prev); + ++prev; + + } + + } + + } + + tokens.push_back(string(match_begin, match_end)); + DEBUGF(stderr, "TOK: %s\n", tokens[tokens.size() - 1].c_str()); + + } + + DEBUGF(stderr, "AFTER all strings\n"); + + if (cur < ende) { + + while (cur < ende) { + + if (isspace(*cur)) { + + auto start = cur; + while (isspace(*cur)) { + + ++cur; + + } + + tokens.push_back(std::string(start, cur)); + DEBUGF(stderr, "WHITESPACE %ld \"%s\"\n", cur - start, + tokens[tokens.size() - 1].c_str()); + + } else if (isalnum(*cur) || *cur == '$' || *cur == '_') { + + auto start = cur; + while (isalnum(*cur) || *cur == '$' || *cur == '_' || *cur == '.' || + *cur == '/') { + + ++cur; + + } + + tokens.push_back(std::string(start, cur)); + DEBUGF(stderr, "IDENTIFIER %ld \"%s\"\n", cur - start, + tokens[tokens.size() - 1].c_str()); + + } else { + + tokens.push_back(std::string(cur, cur + 1)); + DEBUGF(stderr, "OTHER \"%c\"\n", *cur); + ++cur; + + } + + } + + } + + IFDEBUG { + + DEBUGF(stderr, "DUMPING TOKENS:\n"); + for (u32 i = 0; i < tokens.size(); ++i) { + + DEBUGF(stderr, "%s", tokens[i].c_str()); + + } + + DEBUGF(stderr, "---------------------------\n"); + + } + + if (tokens.size() < AUTOTOKENS_SIZE_MIN) { + + file_mapping[fn] = NULL; + s = NULL; + DEBUGF(stderr, "too few tokens\n"); + return 1; + + } + + /* Now we transform the tokens into an ID list and saved that */ + + structure = new vector<u32>(); + u32 id; + + for (u32 i = 0; i < tokens.size(); ++i) { + + if ((id = token_to_id[tokens[i]]) == 0) { + + // First time we see this token, add it to the list + token_to_id[tokens[i]] = current_id; + id_to_token[current_id] = tokens[i]; + structure->push_back(current_id); + ++current_id; + + } else { + + structure->push_back(id); + + } + + } + + // save the token structure to the file mapping + file_mapping[fn] = structure; + id_mapping[valid_structures] = structure; + ++valid_structures; + s = structure; + all_structure_items += structure->size(); + + // we are done! + DEBUGF(stderr, "DONE! We have %lu tokens in the structure\n", + structure->size()); + + } else { + + if (entry->second == NULL) { + + DEBUGF(stderr, "Skipping %s\n", filename); + s = NULL; + return 1; + + } + + s = entry->second; + DEBUGF(stderr, "OK %s\n", filename); + + } + + return 1; // we always fuzz unless non-ascii or too small + +} + +extern "C" my_mutator_t *afl_custom_init(afl_state *afl, unsigned int seed) { + + (void)(seed); + my_mutator_t *data = (my_mutator_t *)calloc(1, sizeof(my_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + if (getenv("AUTOTOKENS_DEBUG")) { debug = 1; } + if (getenv("AUTOTOKENS_AUTO_DISABLE")) { auto_disable = 1; } + if (getenv("AUTOTOKENS_ONLY_FAV")) { only_fav = 1; } + if (getenv("AUTOTOKENS_CREATE_FROM_THIN_AIR")) { create_from_thin_air = 1; } + + if (getenv("AUTOTOKENS_LEARN_DICT")) { + + learn_dictionary_tokens = atoi(getenv("AUTOTOKENS_LEARN_DICT")); + if (learn_dictionary_tokens < 0 || learn_dictionary_tokens > 2) { + + learn_dictionary_tokens = AUTOTOKENS_LEARN_DICT; + + } + + } + + if (getenv("AUTOTOKENS_FUZZ_COUNT_SHIFT")) { + + fuzz_count_shift = atoi(getenv("AUTOTOKENS_FUZZ_COUNT_SHIFT")); + if (fuzz_count_shift < 0 || fuzz_count_shift > 16) { fuzz_count_shift = 0; } + + } + + if (getenv("AUTOTOKENS_CHANGE_MIN")) { + + change_min = atoi(getenv("AUTOTOKENS_CHANGE_MIN")); + if (change_min < 1 || change_min > 256) { + + change_min = AUTOTOKENS_CHANGE_MIN; + + } + + } + + if (getenv("AUTOTOKENS_CHANGE_MAX")) { + + change_max = atoi(getenv("AUTOTOKENS_CHANGE_MAX")); + if (change_max < 1 || change_max > 4096) { + + change_max = AUTOTOKENS_CHANGE_MAX; + + } + + } + + if (change_max < change_min) { change_max = change_min + 1; } + + if (getenv("AUTOTOKENS_COMMENT")) { + + char buf[256]; + snprintf(buf, sizeof(buf), "(%s.*)([\r\n]?)", getenv("AUTOTOKENS_COMMENT")); + regex_comment_custom = new regex(buf, regex::optimize); + + } + + data->afl = afl_ptr = afl; + + // set common whitespace tokens + // we deliberately do not put uncommon ones here to these will count as + // identifier tokens. + token_to_id[" "] = current_id; + id_to_token[current_id] = " "; + ++current_id; + token_to_id["\t"] = current_id; + id_to_token[current_id] = "\t"; + ++current_id; + token_to_id["\n"] = current_id; + id_to_token[current_id] = "\n"; + ++current_id; + token_to_id["\r\n"] = current_id; + id_to_token[current_id] = "\r\n"; + ++current_id; + token_to_id[" \n"] = current_id; + id_to_token[current_id] = " \n"; + ++current_id; + token_to_id[" "] = current_id; + id_to_token[current_id] = " "; + ++current_id; + token_to_id["\t\t"] = current_id; + id_to_token[current_id] = "\t\t"; + ++current_id; + token_to_id["\n\n"] = current_id; + id_to_token[current_id] = "\n\n"; + ++current_id; + token_to_id["\r\n\r\n"] = current_id; + id_to_token[current_id] = "\r\n\r\n"; + ++current_id; + token_to_id[" "] = current_id; + id_to_token[current_id] = " "; + ++current_id; + token_to_id["\t\t\t\t"] = current_id; + id_to_token[current_id] = "\t\t\t\t"; + ++current_id; + token_to_id["\n\n\n\n"] = current_id; + id_to_token[current_id] = "\n\n\n\n"; + ++current_id; + whitespace_ids = current_id; + token_to_id["\""] = current_id; + id_to_token[current_id] = "\""; + ++current_id; + token_to_id["'"] = current_id; + id_to_token[current_id] = "'"; + ++current_id; + + return data; + +} + +extern "C" void afl_custom_splice_optout(my_mutator_t *data) { + + (void)(data); + +} + +extern "C" void afl_custom_deinit(my_mutator_t *data) { + + /* we use this to print statistics at exit :-) + needs to be stderr as stdout is filtered */ + + if (module_disabled) { return; } + + fprintf(stderr, + "\n\nAutotoken mutator statistics:\n" + " Number of all seen tokens: %u\n" + " Number of input structures: %u\n" + " Number of all items in structures: %llu\n" + " Number of total fuzzes: %llu\n\n", + current_id - 1, valid_structures, all_structure_items, fuzz_count); + + free(data); + +} + diff --git a/custom_mutators/examples/custom_send.c b/custom_mutators/examples/custom_send.c index ffea927e..7de72819 100644 --- a/custom_mutators/examples/custom_send.c +++ b/custom_mutators/examples/custom_send.c @@ -1,7 +1,14 @@ +// +// This is an example on how to use afl_custom_send +// It writes each mutated data set to /tmp/foo +// You can modify this to send to IPC, shared memory, etc. +// // cc -O3 -fPIC -shared -g -o custom_send.so -I../../include custom_send.c // cd ../.. // afl-cc -o test-instr test-instr.c -// afl-fuzz -i in -o out -- ./test-instr -f /tmp/foo +// AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/examples/custom_send.so \ +// afl-fuzz -i in -o out -- ./test-instr -f /tmp/foo +// #include "custom_mutator_helpers.h" diff --git a/custom_mutators/examples/example.c b/custom_mutators/examples/example.c index 3f299508..e680ec8e 100644 --- a/custom_mutators/examples/example.c +++ b/custom_mutators/examples/example.c @@ -6,7 +6,7 @@ Dominik Maier <mail@dmnk.co> */ -// You need to use -I /path/to/AFLplusplus/include +// You need to use -I/path/to/AFLplusplus/include -I. #include "custom_mutator_helpers.h" #include <stdint.h> @@ -118,6 +118,8 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, } + if (max_size > mutated_size) { mutated_size = max_size; } + *out_buf = mutated_out; return mutated_size; diff --git a/custom_mutators/examples/post_library_gif.so.c b/custom_mutators/examples/post_library_gif.so.c index 9cd224f4..3cb018a6 100644 --- a/custom_mutators/examples/post_library_gif.so.c +++ b/custom_mutators/examples/post_library_gif.so.c @@ -129,8 +129,8 @@ size_t afl_custom_post_process(post_state_t *data, unsigned char *in_buf, /* Allocate memory for new buffer, reusing previous allocation if possible. Note we have to use afl-fuzz's own realloc! - Note that you should only do this if you need to grow the buffer, - otherwise work with in_buf, and assign it to *out_buf instead. */ + We use afl_realloc because it is effective. + You can also work within in_buf, and assign it to *out_buf. */ *out_buf = afl_realloc(out_buf, len); diff --git a/docs/Changelog.md b/docs/Changelog.md index eee88a51..8f71fd83 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -6,10 +6,13 @@ ### Version ++4.06a (dev) - afl-fuzz: - ensure temporary file descriptor is closed when not used + - added `AFL_NO_WARN_INSTABILITY` - afl-cc: - add CFI sanitizer variant to gcc targets - llvm 16 support (thanks to @devnexen!) - support llvm 15 native pcguard changes + - new custom module: autotoken, grammar free fuzzer for text inputs + - LTO autoken and llvm_mode: added AFL_LLVM_DICT2FILE_NO_MAIN support - better sanitizer default options support for all tools - unicorn_mode: updated and minor issues fixed - frida_mode: fix issue on MacOS diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index 4ffeda7a..82131c92 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -48,6 +48,7 @@ C/C++: ```c void *afl_custom_init(afl_state_t *afl, unsigned int seed); unsigned int afl_custom_fuzz_count(void *data, const unsigned char *buf, size_t buf_size); +void afl_custom_splice_optout(void *data); size_t afl_custom_fuzz(void *data, unsigned char *buf, size_t buf_size, unsigned char **out_buf, unsigned char *add_buf, size_t add_buf_size, size_t max_size); const char *afl_custom_describe(void *data, size_t max_description_len); size_t afl_custom_post_process(void *data, unsigned char *buf, size_t buf_size, unsigned char **out_buf); @@ -72,6 +73,9 @@ def init(seed): def fuzz_count(buf): return cnt +def splice_optout() + pass + def fuzz(buf, add_buf, max_size): return mutated_out @@ -132,6 +136,13 @@ def deinit(): # optional for Python for a specific queue entry, use this function. This function is most useful if `AFL_CUSTOM_MUTATOR_ONLY` is **not** used. +- `splice_optout` (optional): + + If this function is present, no splicing target is passed to the `fuzz` + function. This saves time if splicing data is not needed by the custom + fuzzing function. + This function is never called, just needs to be present to activate. + - `fuzz` (optional): This method performs custom mutations on a given input. It also accepts an @@ -139,6 +150,7 @@ def deinit(): # optional for Python sense to use it. You would only skip this if `post_process` is used to fix checksums etc. so if you are using it, e.g., as a post processing library. Note that a length > 0 *must* be returned! + The returned output buffer is under **your** memory management! - `describe` (optional): diff --git a/docs/env_variables.md b/docs/env_variables.md index 22a5c386..6cd4104b 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -129,6 +129,9 @@ subset of the settings discussed in section 1, with the exception of: write all constant string comparisons to this file to be used later with afl-fuzz' `-x` option. + - An option to `AFL_LLVM_DICT2FILE` is `AFL_LLVM_DICT2FILE_NO_MAIN=1` which + skill not parse `main()`. + - `TMPDIR` and `AFL_KEEP_ASSEMBLY`, since no temporary assembly files are created. @@ -354,6 +357,9 @@ checks or alter some of the more exotic semantics of the tool: - Setting `AFL_KEEP_TIMEOUTS` will keep longer running inputs if they reach new coverage + - On the contrary, if you are not interested in any timeouts, you can set + `AFL_IGNORE_TIMEOUTS` to get a bit of speed instead. + - `AFL_EXIT_ON_SEED_ISSUES` will restore the vanilla afl-fuzz behavior which does not allow crashes or timeout seeds in the initial -i corpus. @@ -474,7 +480,10 @@ checks or alter some of the more exotic semantics of the tool: output from afl-fuzz is redirected to a file or to a pipe. - Setting `AFL_NO_STARTUP_CALIBRATION` will skip the initial calibration - of all starting seeds, and start fuzzing at once. + of all starting seeds, and start fuzzing at once. Use with care, this + degrades the fuzzing performance! + + - Setting `AFL_NO_WARN_INSTABILITY` will suppress instability warnings. - In QEMU mode (-Q) and FRIDA mode (-O), `AFL_PATH` will be searched for afl-qemu-trace and afl-frida-trace.so. diff --git a/docs/fuzzing_in_depth.md b/docs/fuzzing_in_depth.md index 87f31a58..f75ca5dc 100644 --- a/docs/fuzzing_in_depth.md +++ b/docs/fuzzing_in_depth.md @@ -534,6 +534,8 @@ dictionaries/FORMAT.dict`. * With `afl-clang-fast`, you can set `AFL_LLVM_DICT2FILE=/full/path/to/new/file.dic` to automatically generate a dictionary during target compilation. + Adding `AFL_LLVM_DICT2FILE_NO_MAIN=1` to not parse main (usually command line + parameter parsing) is often a good idea too. * You also have the option to generate a dictionary yourself during an independent run of the target, see [utils/libtokencap/README.md](../utils/libtokencap/README.md). @@ -628,7 +630,8 @@ If you have a large corpus, a corpus from a previous run or are fuzzing in a CI, then also set `export AFL_CMPLOG_ONLY_NEW=1` and `export AFL_FAST_CAL=1`. If the queue in the CI is huge and/or the execution time is slow then you can also add `AFL_NO_STARTUP_CALIBRATION=1` to skip the initial queue calibration -phase and start fuzzing at once. +phase and start fuzzing at once - but only do this if the calibration phase +would be too long for your fuzz run time. You can also use different fuzzers. If you are using AFL spinoffs or AFL conforming fuzzers, then just use the same -o directory and give it a unique @@ -914,7 +917,8 @@ normal fuzzing campaigns as these are much shorter runnings. If the queue in the CI is huge and/or the execution time is slow then you can also add `AFL_NO_STARTUP_CALIBRATION=1` to skip the initial queue calibration -phase and start fuzzing at once. +phase and start fuzzing at once. But only do that if the calibration time is +too long for your overall available fuzz run time. 1. Always: * LTO has a much longer compile time which is diametrical to short fuzzing - @@ -935,7 +939,7 @@ phase and start fuzzing at once. 3. Also randomize the afl-fuzz runtime options, e.g.: * 65% for `AFL_DISABLE_TRIM` * 50% for `AFL_KEEP_TIMEOUTS` - * 50% use a dictionary generated by `AFL_LLVM_DICT2FILE` + * 50% use a dictionary generated by `AFL_LLVM_DICT2FILE` + `AFL_LLVM_DICT2FILE_NO_MAIN=1` * 40% use MOpt (`-L 0`) * 40% for `AFL_EXPAND_HAVOC_NOW` * 20% for old queue processing (`-Z`) diff --git a/frida_mode/test/cmplog/GNUmakefile b/frida_mode/test/cmplog/GNUmakefile index bcaff42d..fca52f82 100644 --- a/frida_mode/test/cmplog/GNUmakefile +++ b/frida_mode/test/cmplog/GNUmakefile @@ -2,8 +2,9 @@ PWD:=$(shell pwd)/ ROOT:=$(PWD)../../../ BUILD_DIR:=$(PWD)build/ +TEST_CMPLOG_BASENAME=compcovtest TEST_CMPLOG_SRC=$(PWD)cmplog.c -TEST_CMPLOG_OBJ=$(BUILD_DIR)compcovtest +TEST_CMPLOG_OBJ=$(BUILD_DIR)$(TEST_CMPLOG_BASENAME) TEST_BIN:=$(PWD)../../build/test @@ -13,7 +14,7 @@ CMP_LOG_INPUT:=$(TEST_DATA_DIR)in QEMU_OUT:=$(BUILD_DIR)qemu-out FRIDA_OUT:=$(BUILD_DIR)frida-out -.PHONY: all 32 clean qemu frida frida-nocmplog format +.PHONY: all 32 clean qemu frida frida-nocmplog frida-unprefixedpath format all: $(TEST_CMPLOG_OBJ) make -C $(ROOT)frida_mode/ @@ -64,6 +65,18 @@ frida-nocmplog: $(TEST_CMPLOG_OBJ) $(CMP_LOG_INPUT) -- \ $(TEST_CMPLOG_OBJ) @@ + +frida-unprefixedpath: $(TEST_CMPLOG_OBJ) $(CMP_LOG_INPUT) + PATH=$(BUILD_DIR) $(ROOT)afl-fuzz \ + -O \ + -i $(TEST_DATA_DIR) \ + -o $(FRIDA_OUT) \ + -c 0 \ + -l 3AT \ + -Z \ + -- \ + $(TEST_CMPLOG_BASENAME) @@ + debug: $(TEST_CMPLOG_OBJ) $(CMP_LOG_INPUT) gdb \ --ex 'set environment LD_PRELOAD=$(ROOT)afl-frida-trace.so' \ diff --git a/frida_mode/test/cmplog/Makefile b/frida_mode/test/cmplog/Makefile index 7ca9a9a5..b84e9218 100644 --- a/frida_mode/test/cmplog/Makefile +++ b/frida_mode/test/cmplog/Makefile @@ -19,6 +19,9 @@ frida: frida-nocmplog: @gmake frida-nocmplog +frida-unprefixedpath: + @gmake frida-unprefixedpath + format: @gmake format diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index edef9207..9bf91faf 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -344,6 +344,7 @@ enum { /* 12 */ PY_FUNC_INTROSPECTION, /* 13 */ PY_FUNC_DESCRIBE, /* 14 */ PY_FUNC_FUZZ_SEND, + /* 15 */ PY_FUNC_SPLICE_OPTOUT, PY_FUNC_COUNT }; @@ -398,7 +399,7 @@ typedef struct afl_env_vars { afl_cycle_schedules, afl_expand_havoc, afl_statsd, afl_cmplog_only_new, afl_exit_on_seed_issues, afl_try_affinity, afl_ignore_problems, afl_keep_timeouts, afl_pizza_mode, afl_no_crash_readme, - afl_no_startup_calibration; + afl_ignore_timeouts, afl_no_startup_calibration, afl_no_warn_instability; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_preload, @@ -495,6 +496,7 @@ typedef struct afl_state { no_unlink, /* do not unlink cur_input */ debug, /* Debug mode */ custom_only, /* Custom mutator only mode */ + custom_splice_optout, /* Custom mutator no splice buffer */ is_main_node, /* if this is the main node */ is_secondary_node, /* if this is a secondary instance */ pizza_is_served; /* pizza mode */ @@ -829,17 +831,29 @@ struct custom_mutator { u32 (*afl_custom_fuzz_count)(void *data, const u8 *buf, size_t buf_size); /** - * Perform custom mutations on a given input + * Opt-out of a splicing input for the fuzz mutator * - * (Optional for now. Required in the future) + * Empty dummy function. It's presence tells afl-fuzz not to pass a + * splice data pointer and len. * * @param data pointer returned in afl_custom_init by this custom mutator + * @noreturn + */ + void (*afl_custom_splice_optout)(void *data); + + /** + * Perform custom mutations on a given input + * + * (Optional) + * + * Getting an add_buf can be skipped by using afl_custom_splice_optout(). + * + * @param[in] data Pointer returned in afl_custom_init by this custom mutator * @param[in] buf Pointer to the input data to be mutated and the mutated * output * @param[in] buf_size Size of the input/output data - * @param[out] out_buf the new buffer. We may reuse *buf if large enough. - * *out_buf = NULL is treated as FATAL. - * @param[in] add_buf Buffer containing the additional test case + * @param[out] out_buf The new buffer, under your memory mgmt. + * @param[in] add_buf Buffer containing an additional test case (splicing) * @param[in] add_buf_size Size of the additional test case * @param[in] max_size Maximum size of the mutated output. The mutation must * not produce data larger than max_size. @@ -1057,6 +1071,7 @@ u8 havoc_mutation_probability_py(void *); u8 queue_get_py(void *, const u8 *); const char *introspection_py(void *); u8 queue_new_entry_py(void *, const u8 *, const u8 *); +void splice_optout(void *); void deinit_py(void *); #endif diff --git a/include/config.h b/include/config.h index a5a4c473..e46f515a 100644 --- a/include/config.h +++ b/include/config.h @@ -364,9 +364,9 @@ * * ***********************************************************/ -/* Call count interval between reseeding the libc PRNG from /dev/urandom: */ +/* Call count interval between reseeding the PRNG from /dev/urandom: */ -#define RESEED_RNG 100000 +#define RESEED_RNG 2500000 /* The default maximum testcase cache size in MB, 0 = disable. A value between 50 and 250 is a good default value. Note that the @@ -491,10 +491,14 @@ #define AFL_TXT_MIN_LEN 12 +/* Maximum length of a queue input to be evaluated for "is_ascii"? */ + +#define AFL_TXT_MAX_LEN 65535 + /* What is the minimum percentage of ascii characters present to be classifed as "is_ascii"? */ -#define AFL_TXT_MIN_PERCENT 94 +#define AFL_TXT_MIN_PERCENT 99 /* How often to perform ASCII mutations 0 = disable, 1-8 are good values */ diff --git a/include/envs.h b/include/envs.h index 46fc796b..cf069a00 100644 --- a/include/envs.h +++ b/include/envs.h @@ -104,6 +104,7 @@ static char *afl_environment_variables[] = { "AFL_HARDEN", "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", "AFL_IGNORE_PROBLEMS", + "AFL_IGNORE_TIMEOUTS", "AFL_IGNORE_UNKNOWN_ENVS", "AFL_IMPORT_FIRST", "AFL_INPUT_LEN_MIN", @@ -133,6 +134,7 @@ static char *afl_environment_variables[] = { "AFL_LLVM_CTX", "AFL_LLVM_CTX_K", "AFL_LLVM_DICT2FILE", + "AFL_LLVM_DICT2FILE_NO_MAIN", "AFL_LLVM_DOCUMENT_IDS", "AFL_LLVM_INSTRIM_LOOPHEAD", "AFL_LLVM_INSTRUMENT", @@ -171,6 +173,7 @@ static char *afl_environment_variables[] = { "AFL_NO_UI", "AFL_NO_PYTHON", "AFL_NO_STARTUP_CALIBRATION", + "AFL_NO_WARN_INSTABILITY", "AFL_UNTRACER_FILE", "AFL_LLVM_USE_TRACE_PC", "AFL_MAP_SIZE", diff --git a/include/forkserver.h b/include/forkserver.h index 35bc1771..50898a08 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -43,7 +43,7 @@ typedef enum NyxReturnValue { Normal, Crash, Asan, - Timout, + Timeout, InvalidWriteToPayload, Error, IoError, diff --git a/instrumentation/README.llvm.md b/instrumentation/README.llvm.md index 9da1b0f6..c0677474 100644 --- a/instrumentation/README.llvm.md +++ b/instrumentation/README.llvm.md @@ -167,6 +167,10 @@ Just specify `AFL_LLVM_DICT2FILE=/absolute/path/file.txt` and during compilation all constant string compare parameters will be written to this file to be used with afl-fuzz' `-x` option. +Adding `AFL_LLVM_DICT2FILE_NO_MAIN=1` will skip parsing `main()` which often +does command line parsing which has string comparisons that are not helpful +for fuzzing. + ## 6) AFL++ Context Sensitive Branch Coverage ### What is this? diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index 231151f5..f82224ed 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -236,6 +236,7 @@ class ModuleSanitizerCoverageLTO // const SpecialCaseList * Allowlist; // const SpecialCaseList * Blocklist; uint32_t autodictionary = 1; + uint32_t autodictionary_no_main = 0; uint32_t inst = 0; uint32_t afl_global_id = 0; uint32_t unhandled = 0; @@ -411,7 +412,8 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( /* Show a banner */ setvbuf(stdout, NULL, _IONBF, 0); - if (getenv("AFL_DEBUG")) debug = 1; + if (getenv("AFL_DEBUG")) { debug = 1; } + if (getenv("AFL_LLVM_DICT2FILE_NO_MAIN")) { autodictionary_no_main = 1; } if ((isatty(2) && !getenv("AFL_QUIET")) || debug) { @@ -503,6 +505,13 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( if (!isInInstrumentList(&F, MNAME) || !F.size()) { continue; } + if (autodictionary_no_main && + (!F.getName().compare("main") || !F.getName().compare("_main"))) { + + continue; + + } + for (auto &BB : F) { for (auto &IN : BB) { diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index d6d6c38c..9871d7f4 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -1518,9 +1518,13 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { _is_sancov = 1; - __afl_auto_first(); - __afl_auto_second(); - __afl_auto_early(); + if (!getenv("AFL_DUMP_MAP_SIZE")) { + + __afl_auto_first(); + __afl_auto_second(); + __afl_auto_early(); + + } if (__afl_debug) { @@ -1534,6 +1538,16 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { if (start == stop || *start) return; + x = getenv("AFL_INST_RATIO"); + if (x) { inst_ratio = (u32)atoi(x); } + + if (!inst_ratio || inst_ratio > 100) { + + fprintf(stderr, "[-] ERROR: Invalid AFL_INST_RATIO (must be 1-100).\n"); + abort(); + + } + // If a dlopen of an instrumented library happens after the forkserver then // we have a problem as we cannot increase the coverage map anymore. if (__afl_already_initialized_forkserver) { @@ -1554,74 +1568,34 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { while (start < stop) { - *(start++) = offset; + if (likely(inst_ratio == 100) || R(100) < inst_ratio) + *start = offset; + else + *start = 0; // write to map[0] if (unlikely(++offset >= __afl_final_loc)) { offset = 4; } } } - } - - x = getenv("AFL_INST_RATIO"); - if (x) { inst_ratio = (u32)atoi(x); } - - if (!inst_ratio || inst_ratio > 100) { - - fprintf(stderr, "[-] ERROR: Invalid AFL_INST_RATIO (must be 1-100).\n"); - abort(); - - } - - /* instrumented code is loaded *after* our forkserver is up. this is a - problem. We cannot prevent collisions then :( */ - /* - if (__afl_already_initialized_forkserver && - __afl_final_loc + 1 + stop - start > __afl_map_size) { - - if (__afl_debug) { - - fprintf(stderr, "Warning: new instrumented code after the forkserver!\n"); - - } - - __afl_final_loc = 2; - - if (1 + stop - start > __afl_map_size) { - - *(start++) = ++__afl_final_loc; - - while (start < stop) { - - if (R(100) < inst_ratio) - *start = ++__afl_final_loc % __afl_map_size; - else - *start = 4; - - start++; - - } - - return; - - } + return; // we are done for this special case } - */ - /* Make sure that the first element in the range is always set - we use that to avoid duplicate calls (which can happen as an artifact of the underlying implementation in LLVM). */ + if (__afl_final_loc < 3) __afl_final_loc = 3; // we skip the first 4 entries + *(start++) = ++__afl_final_loc; while (start < stop) { - if (R(100) < inst_ratio) + if (likely(inst_ratio == 100) || R(100) < inst_ratio) *start = ++__afl_final_loc; else - *start = 4; + *start = 0; // write to map[0] start++; diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index bbbbe32c..97f1d47f 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -182,7 +182,7 @@ bool AFLdict2filePass::runOnModule(Module &M) { DenseMap<Value *, std::string *> valueMap; char *ptr; - int found = 0; + int found = 0, handle_main = 1; /* Show a banner */ setvbuf(stdout, NULL, _IONBF, 0); @@ -192,10 +192,14 @@ bool AFLdict2filePass::runOnModule(Module &M) { SAYF(cCYA "afl-llvm-dict2file" VERSION cRST " by Marc \"vanHauser\" Heuse <mh@mh-sec.de>\n"); - } else + } else { be_quiet = 1; + } + + if (getenv("AFL_LLVM_DICT2FILE_NO_MAIN")) { handle_main = 0; } + scanForDangerousFunctions(&M); ptr = getenv("AFL_LLVM_DICT2FILE"); @@ -210,7 +214,14 @@ bool AFLdict2filePass::runOnModule(Module &M) { for (auto &F : M) { - if (isIgnoreFunction(&F)) continue; + if (!handle_main && + (!F.getName().compare("main") || !F.getName().compare("_main"))) { + + continue; + + } + + if (isIgnoreFunction(&F)) { continue; } if (!isInInstrumentList(&F, MNAME) || !F.size()) { continue; } /* Some implementation notes. diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index dd7b09a6..8a07610c 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -1152,10 +1152,14 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { b_op1 = SelectInst::Create(isMzero_op1, ConstantInt::get(intType, PlusZero), bpre_op1); #if LLVM_MAJOR >= 16 - isMzero_op0->insertInto(nonan_bb, BasicBlock::iterator(nonan_bb->getTerminator())); - isMzero_op1->insertInto(nonan_bb, BasicBlock::iterator(nonan_bb->getTerminator())); - b_op0->insertInto(nonan_bb, BasicBlock::iterator(nonan_bb->getTerminator())); - b_op1->insertInto(nonan_bb, BasicBlock::iterator(nonan_bb->getTerminator())); + isMzero_op0->insertInto(nonan_bb, + BasicBlock::iterator(nonan_bb->getTerminator())); + isMzero_op1->insertInto(nonan_bb, + BasicBlock::iterator(nonan_bb->getTerminator())); + b_op0->insertInto(nonan_bb, + BasicBlock::iterator(nonan_bb->getTerminator())); + b_op1->insertInto(nonan_bb, + BasicBlock::iterator(nonan_bb->getTerminator())); #else nonan_bb->getInstList().insert( BasicBlock::iterator(nonan_bb->getTerminator()), isMzero_op0); @@ -1192,7 +1196,8 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { t_s0->insertInto(nonan_bb, BasicBlock::iterator(nonan_bb->getTerminator())); s_s1->insertInto(nonan_bb, BasicBlock::iterator(nonan_bb->getTerminator())); t_s1->insertInto(nonan_bb, BasicBlock::iterator(nonan_bb->getTerminator())); - icmp_sign_bit->insertInto(nonan_bb, BasicBlock::iterator(nonan_bb->getTerminator())); + icmp_sign_bit->insertInto(nonan_bb, + BasicBlock::iterator(nonan_bb->getTerminator())); #else nonan_bb->getInstList().insert( BasicBlock::iterator(nonan_bb->getTerminator()), s_s0); @@ -1239,8 +1244,10 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { Instruction::LShr, b_op1, ConstantInt::get(b_op1->getType(), shiftR_exponent)); #if LLVM_MAJOR >= 16 - s_e0->insertInto(signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); - s_e1->insertInto(signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); + s_e0->insertInto(signequal_bb, + BasicBlock::iterator(signequal_bb->getTerminator())); + s_e1->insertInto(signequal_bb, + BasicBlock::iterator(signequal_bb->getTerminator())); #else signequal_bb->getInstList().insert( BasicBlock::iterator(signequal_bb->getTerminator()), s_e0); @@ -1251,15 +1258,16 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { t_e0 = new TruncInst(s_e0, IntExponentTy); t_e1 = new TruncInst(s_e1, IntExponentTy); #if LLVM_MAJOR >= 16 - t_e0->insertInto(signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); - t_e1->insertInto(signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); + t_e0->insertInto(signequal_bb, + BasicBlock::iterator(signequal_bb->getTerminator())); + t_e1->insertInto(signequal_bb, + BasicBlock::iterator(signequal_bb->getTerminator())); #else signequal_bb->getInstList().insert( BasicBlock::iterator(signequal_bb->getTerminator()), t_e0); signequal_bb->getInstList().insert( BasicBlock::iterator(signequal_bb->getTerminator()), t_e1); #endif - if (sizeInBits - precision < exTySizeBytes * 8) { @@ -1270,8 +1278,10 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { Instruction::And, t_e1, ConstantInt::get(t_e1->getType(), mask_exponent)); #if LLVM_MAJOR >= 16 - m_e0->insertInto(signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); - m_e1->insertInto(signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); + m_e0->insertInto(signequal_bb, + BasicBlock::iterator(signequal_bb->getTerminator())); + m_e1->insertInto(signequal_bb, + BasicBlock::iterator(signequal_bb->getTerminator())); #else signequal_bb->getInstList().insert( BasicBlock::iterator(signequal_bb->getTerminator()), m_e0); @@ -1312,7 +1322,8 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { icmp_exponents_equal = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, m_e0, m_e1); #if LLVM_MAJOR >= 16 - icmp_exponents_equal->insertInto(signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); + icmp_exponents_equal->insertInto( + signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); #else signequal_bb->getInstList().insert( BasicBlock::iterator(signequal_bb->getTerminator()), @@ -1332,7 +1343,9 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { icmp_exponent = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_UGT, m_e0, m_e1); #if LLVM_MAJOR >= 16 - icmp_exponent->insertInto(signequal2_bb, BasicBlock::iterator(signequal2_bb->getTerminator())); + icmp_exponent->insertInto( + signequal2_bb, + BasicBlock::iterator(signequal2_bb->getTerminator())); #else signequal2_bb->getInstList().insert( BasicBlock::iterator(signequal2_bb->getTerminator()), @@ -1346,7 +1359,8 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { icmp_exponents_equal = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, m_e0, m_e1); #if LLVM_MAJOR >= 16 - icmp_exponents_equal->insertInto(signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); + icmp_exponents_equal->insertInto( + signequal_bb, BasicBlock::iterator(signequal_bb->getTerminator())); #else signequal_bb->getInstList().insert( BasicBlock::iterator(signequal_bb->getTerminator()), @@ -1366,7 +1380,9 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { icmp_exponent = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULT, m_e0, m_e1); #if LLVM_MAJOR >= 16 - icmp_exponent->insertInto(signequal2_bb, BasicBlock::iterator(signequal2_bb->getTerminator())); + icmp_exponent->insertInto( + signequal2_bb, + BasicBlock::iterator(signequal2_bb->getTerminator())); #else signequal2_bb->getInstList().insert( BasicBlock::iterator(signequal2_bb->getTerminator()), @@ -1381,7 +1397,8 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { } #if LLVM_MAJOR >= 16 - icmp_exponent_result->insertInto(signequal2_bb, BasicBlock::iterator(signequal2_bb->getTerminator())); + icmp_exponent_result->insertInto( + signequal2_bb, BasicBlock::iterator(signequal2_bb->getTerminator())); #else signequal2_bb->getInstList().insert( BasicBlock::iterator(signequal2_bb->getTerminator()), @@ -1437,8 +1454,10 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { Instruction::And, b_op1, ConstantInt::get(b_op1->getType(), mask_fraction)); #if LLVM_MAJOR >= 16 - m_f0->insertInto(middle_bb, BasicBlock::iterator(middle_bb->getTerminator())); - m_f1->insertInto(middle_bb, BasicBlock::iterator(middle_bb->getTerminator())); + m_f0->insertInto(middle_bb, + BasicBlock::iterator(middle_bb->getTerminator())); + m_f1->insertInto(middle_bb, + BasicBlock::iterator(middle_bb->getTerminator())); #else middle_bb->getInstList().insert( BasicBlock::iterator(middle_bb->getTerminator()), m_f0); @@ -1451,8 +1470,10 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { t_f0 = new TruncInst(m_f0, IntFractionTy); t_f1 = new TruncInst(m_f1, IntFractionTy); #if LLVM_MAJOR >= 16 - t_f0->insertInto(middle_bb, BasicBlock::iterator(middle_bb->getTerminator())); - t_f1->insertInto(middle_bb, BasicBlock::iterator(middle_bb->getTerminator())); + t_f0->insertInto(middle_bb, + BasicBlock::iterator(middle_bb->getTerminator())); + t_f1->insertInto(middle_bb, + BasicBlock::iterator(middle_bb->getTerminator())); #else middle_bb->getInstList().insert( BasicBlock::iterator(middle_bb->getTerminator()), t_f0); @@ -1474,8 +1495,10 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { t_f0 = new TruncInst(b_op0, IntFractionTy); t_f1 = new TruncInst(b_op1, IntFractionTy); #if LLVM_MAJOR >= 16 - t_f0->insertInto(middle_bb, BasicBlock::iterator(middle_bb->getTerminator())); - t_f1->insertInto(middle_bb, BasicBlock::iterator(middle_bb->getTerminator())); + t_f0->insertInto(middle_bb, + BasicBlock::iterator(middle_bb->getTerminator())); + t_f1->insertInto(middle_bb, + BasicBlock::iterator(middle_bb->getTerminator())); #else middle_bb->getInstList().insert( BasicBlock::iterator(middle_bb->getTerminator()), t_f0); @@ -1503,7 +1526,8 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { icmp_fraction_result = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, t_f0, t_f1); #if LLVM_MAJOR >= 16 - icmp_fraction_result->insertInto(middle2_bb, BasicBlock::iterator(middle2_bb->getTerminator())); + icmp_fraction_result->insertInto( + middle2_bb, BasicBlock::iterator(middle2_bb->getTerminator())); #else middle2_bb->getInstList().insert( BasicBlock::iterator(middle2_bb->getTerminator()), @@ -1516,7 +1540,8 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { icmp_fraction_result = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, t_f0, t_f1); #if LLVM_MAJOR >= 16 - icmp_fraction_result->insertInto(middle2_bb, BasicBlock::iterator(middle2_bb->getTerminator())); + icmp_fraction_result->insertInto( + middle2_bb, BasicBlock::iterator(middle2_bb->getTerminator())); #else middle2_bb->getInstList().insert( BasicBlock::iterator(middle2_bb->getTerminator()), @@ -1542,13 +1567,13 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { if (FcmpInst->getPredicate() == CmpInst::FCMP_OGT || FcmpInst->getPredicate() == CmpInst::FCMP_UGT) { - icmp_fraction_result = CmpInst::Create( - Instruction::ICmp, CmpInst::ICMP_ULT, t_f0, t_f1); - icmp_fraction_result2 = CmpInst::Create( - Instruction::ICmp, CmpInst::ICMP_UGT, t_f0, t_f1); + icmp_fraction_result = + CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULT, t_f0, t_f1); + icmp_fraction_result2 = + CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_UGT, t_f0, t_f1); #if LLVM_MAJOR >= 16 - icmp_fraction_result->insertInto(negative_bb, negative_bb->end()); - icmp_fraction_result2->insertInto(positive_bb, negative_bb->end()); + icmp_fraction_result->insertInto(negative_bb, negative_bb->end()); + icmp_fraction_result2->insertInto(positive_bb, negative_bb->end()); #else negative_bb->getInstList().push_back(icmp_fraction_result); positive_bb->getInstList().push_back(icmp_fraction_result2); @@ -1556,13 +1581,13 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { } else { - icmp_fraction_result = CmpInst::Create( - Instruction::ICmp, CmpInst::ICMP_UGT, t_f0, t_f1); - icmp_fraction_result2 = CmpInst::Create( - Instruction::ICmp, CmpInst::ICMP_ULT, t_f0, t_f1); + icmp_fraction_result = + CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_UGT, t_f0, t_f1); + icmp_fraction_result2 = + CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULT, t_f0, t_f1); #if LLVM_MAJOR >= 16 - icmp_fraction_result->insertInto(negative_bb, negative_bb->end()); - icmp_fraction_result2->insertInto(positive_bb, negative_bb->end()); + icmp_fraction_result->insertInto(negative_bb, negative_bb->end()); + icmp_fraction_result2->insertInto(positive_bb, negative_bb->end()); #else negative_bb->getInstList().push_back(icmp_fraction_result); positive_bb->getInstList().push_back(icmp_fraction_result2); @@ -1581,7 +1606,8 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { PN2->addIncoming(icmp_fraction_result, negative_bb); PN2->addIncoming(icmp_fraction_result2, positive_bb); #if LLVM_MAJOR >= 16 - PN2->insertInto(middle2_bb, BasicBlock::iterator(middle2_bb->getTerminator())); + PN2->insertInto(middle2_bb, + BasicBlock::iterator(middle2_bb->getTerminator())); #else middle2_bb->getInstList().insert( BasicBlock::iterator(middle2_bb->getTerminator()), PN2); diff --git a/qemu_mode/README.deferred_initialization_example.md b/qemu_mode/README.deferred_initialization_example.md new file mode 100644 index 00000000..d940d6b5 --- /dev/null +++ b/qemu_mode/README.deferred_initialization_example.md @@ -0,0 +1,201 @@ +# Fuzz ARM32 Python Native Extensions in Binary-only Mode (LLVM fork-based) + +This is an example on how to fuzz Python native extensions in LLVM mode with deferred initialization on ARM32. + +We use Ubuntu x86_64 to run AFL++ and an Alpine ARMv7 Chroot to build the fuzzing target. + +Check [Resources](#resources) for the code used in this example. + +## Setup Alpine ARM Chroot on your x86_64 Linux Host + +### Use systemd-nspawn + +1. Install `qemu-user-binfmt`, `qemu-user-static` and `systemd-container` dependencies. +2. Restart the systemd-binfmt service: `systemctl restart systemd-binfmt.service` +3. Download an Alpine ARM RootFS from https://alpinelinux.org/downloads/ +4. Create a new `alpine_sysroot` folder and extract: `tar xfz alpine-minirootfs-3.17.1-armv7.tar.gz -C alpine_sysroot/` +5. Copy `qemu-arm-static` to Alpine's RootFS: `cp $(which qemu-arm-static) ./alpine/usr/bin/` +6. Chroot into the container: `sudo systemd-nspawn -D alpine/ --bind-ro=/etc/resolv.conf` +7. Install dependencies: `apk update && apk add build-base musl-dev clang15 python3 python3-dev py3-pip` +8. Exit the container with `exit` + +### Alternatively use Docker + +1. Install `qemu-user-binfmt` and `qemu-user-static` +2. Run Qemu container: ```$ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes``` +3. Run Alpine container: ```$ docker run -it --rm arm32v7/alpine sh``` + +## Build AFL++ Qemu Mode with ARM Support + +First, build AFL++ as described [here](https://github.com/AFLplusplus/AFLplusplus/blob/dev/docs/INSTALL.md). Then, run the Qemu build script: + +```bash +cd qemu_mode && CPU_TARGET=arm ./build_qemu_support.sh +``` + +## Compile and Build the Fuzzing Project +Build the native extension and the fuzzing harness for ARM using the Alpine container (check [Resources](#resources) for the code): +```bash +ALPINE_ROOT=<your-alpine-sysroot-directory> +FUZZ=<your-path-to-the-code> +sudo systemd-nspawn -D $ALPINE_ROOT --bind=$FUZZ:/fuzz +CC=$(which clang) CFLAGS="-g" LDSHARED="clang -shared" python3 -m pip install /fuzz +clang $(python3-config --embed --cflags) $(python3-config --embed --ldflags) -o /fuzz/fuzz_harness /fuzz/fuzz_harness.c +exit +``` + +Manually trigger bug: +```bash +echo -n "FUZZ" | qemu-arm-static -L $ALPINE_ROOT $FUZZ/fuzz_harness +``` + +## Run AFL++ +Make sure to start the forkserver *after* loading all the shared objects by setting the `AFL_ENTRYPOINT` environment variable (see [here](https://aflplus.plus/docs/env_variables/#5-settings-for-afl-qemu-trace) for details): + +Choose an address just before the `while()` loop, for example: +```bash +qemu-arm-static -L $ALPINE_ROOT $ALPINE_ROOT/usr/bin/objdump -d $FUZZ/fuzz_harness | grep -A 1 "PyObject_GetAttrString" + +00000584 <PyObject_GetAttrString@plt>: + 584: e28fc600 add ip, pc, #0, 12 +-- + 7c8: ebffff6d bl 584 <PyObject_GetAttrString@plt> + 7cc: e58d0008 str r0, [sp, #8] +... +``` + +Check Qemu memory maps using the instructions from [here](https://aflplus.plus/docs/tutorials/libxml2_tutorial/): +>The binary is position independent and QEMU persistent needs the real addresses, not the offsets. Fortunately, QEMU loads PIE executables at a fixed address, 0x4000000000 for x86_64. +> +> We can check it using `AFL_QEMU_DEBUG_MAPS`. You don’t need this step if your binary is not PIE. + +Setup Python environment variables and run `afl-qemu-trace`: +```bash +PYTHONPATH=$ALPINE_ROOT/usr/lib/python3.10/ PYTHONHOME=$ALPINE_ROOT/usr/bin/ QEMU_LD_PREFIX=$ALPINE_ROOT AFL_QEMU_DEBUG_MAPS=1 afl-qemu-trace $FUZZ/fuzz_harness + +... +40000000-40001000 r-xp 00000000 103:03 8002276 fuzz_harness +40001000-4001f000 ---p 00000000 00:00 0 +4001f000-40020000 r--p 0000f000 103:03 8002276 fuzz_harness +40020000-40021000 rw-p 00010000 103:03 8002276 fuzz_harness +40021000-40022000 ---p 00000000 00:00 0 +40022000-40023000 rw-p 00000000 00:00 0 +``` + +Finally, setup Qemu environment variables... +```bash +export QEMU_SET_ENV=PYTHONPATH=$ALPINE_ROOT/usr/lib/python310.zip:$ALPINE_ROOT/usr/lib/python3.10:$ALPINE_ROOT/usr/lib/python3.10/lib-dynload:$ALPINE_ROOT/usr/lib/python3.10/site-packages,PYTHONHOME=$ALPINE_ROOT/usr/bin/ +export QEMU_LD_PREFIX=$ALPINE_ROOT +``` + +... and run AFL++: +```bash +mkdir -p $FUZZ/in && echo -n "FU" > $FUZZ/in/seed +AFL_ENTRYPOINT=0x400007cc afl-fuzz -i $FUZZ/in -o $FUZZ/out -Q -- $FUZZ/fuzz_harness +``` + +## Resources + +### setup.py + +```python +from distutils.core import setup, Extension + +module = Extension("memory", sources=["fuzz_target.c"]) + +setup( + name="memory", + version="1.0", + description='A simple "BOOM!" extension', + ext_modules=[module], +) +``` + +### fuzz_target.c + +```c +#define PY_SSIZE_T_CLEAN +#include <Python.h> + +#pragma clang optimize off + +static PyObject *corruption(PyObject* self, PyObject* args) { + char arr[3]; + Py_buffer name; + + if (!PyArg_ParseTuple(args, "y*", &name)) + return NULL; + + if (name.buf != NULL) { + if (strcmp(name.buf, "FUZZ") == 0) { + arr[0] = 'B'; + arr[1] = 'O'; + arr[2] = 'O'; + arr[3] = 'M'; + } + } + + PyBuffer_Release(&name); + Py_RETURN_NONE; +} + +static PyMethodDef MemoryMethods[] = { + {"corruption", corruption, METH_VARARGS, "BOOM!"}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef memory_module = { + PyModuleDef_HEAD_INIT, + "memory", + "BOOM!", + -1, + MemoryMethods +}; + +PyMODINIT_FUNC PyInit_memory(void) { + return PyModule_Create(&memory_module); +} +``` + +### fuzz_harness.c + +```c +#include <Python.h> + +#pragma clang optimize off + +int main(int argc, char **argv) { + unsigned char buf[1024000]; + ssize_t size; + + Py_Initialize(); + PyObject* name = PyUnicode_DecodeFSDefault("memory"); + PyObject* module = PyImport_Import(name); + Py_DECREF(name); + + if (module != NULL) { + PyObject* corruption_func = PyObject_GetAttrString(module, "corruption"); + + while ((size = read(0, buf, sizeof(buf))) > 0 ? 1 : 0) { + PyObject* arg = PyBytes_FromStringAndSize((char *)buf, size); + + if (arg != NULL) { + PyObject* res = PyObject_CallFunctionObjArgs(corruption_func, arg, NULL); + + if (res != NULL) { + Py_XDECREF(res); + } + + Py_DECREF(arg); + } + } + + Py_DECREF(corruption_func); + Py_DECREF(module); + } + + // Py_Finalize() leaks memory on certain Python versions (see https://bugs.python.org/issue1635741) + // Py_Finalize(); + return 0; +} +``` diff --git a/qemu_mode/README.md b/qemu_mode/README.md index 4ed2f298..92038737 100644 --- a/qemu_mode/README.md +++ b/qemu_mode/README.md @@ -66,6 +66,8 @@ allows to move the forkserver to a different part, e.g., just before the file is opened (e.g., way after command line parsing and config file loading, etc.) which can be a huge speed improvement. +For an example, see [README.deferred_initialization_example.md](README.deferred_initialization_example.md). + ## 4) Persistent mode AFL++'s QEMU mode now supports also persistent mode for x86, x86_64, arm, and diff --git a/src/afl-cc.c b/src/afl-cc.c index 7c3682fb..7b059d40 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -2041,6 +2041,8 @@ int main(int argc, char **argv, char **envp) { " AFL_LLVM_DICT2FILE: generate an afl dictionary based on found " "comparisons\n" + " AFL_LLVM_DICT2FILE_NO_MAIN: skip parsing main() for the " + "dictionary\n" " AFL_LLVM_LAF_ALL: enables all LAF splits/transforms\n" " AFL_LLVM_LAF_SPLIT_COMPARES: enable cascaded comparisons\n" " AFL_LLVM_LAF_SPLIT_COMPARES_BITW: size limit (default 8)\n" @@ -2128,7 +2130,8 @@ int main(int argc, char **argv, char **envp) { "defaults.\n" "Recommended is afl-clang-lto with AFL_LLVM_CMPLOG or afl-clang-fast " "with\n" - "AFL_LLVM_CMPLOG and AFL_LLVM_DICT2FILE.\n\n"); + "AFL_LLVM_CMPLOG and " + "AFL_LLVM_DICT2FILE+AFL_LLVM_DICT2FILE_NO_MAIN.\n\n"); exit(1); diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 89d01460..50dc7a26 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -59,7 +59,7 @@ static list_t fsrv_list = {.element_prealloc_count = 0}; static void fsrv_exec_child(afl_forkserver_t *fsrv, char **argv) { - if (fsrv->qemu_mode || fsrv->cs_mode) { + if (fsrv->qemu_mode || fsrv->frida_mode || fsrv->cs_mode) { setenv("AFL_DISABLE_LLVM_INSTRUMENTATION", "1", 0); @@ -1370,7 +1370,7 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, case Crash: case Asan: return FSRV_RUN_CRASH; - case Timout: + case Timeout: return FSRV_RUN_TMOUT; case InvalidWriteToPayload: /* ??? */ diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 485b82db..b4e9537e 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -457,6 +457,12 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { if (unlikely(len == 0)) { return 0; } + if (unlikely(fault == FSRV_RUN_TMOUT && afl->afl_env.afl_ignore_timeouts)) { + + return 0; + + } + u8 fn[PATH_MAX]; u8 *queue_fn = ""; u8 new_bits = 0, keeping = 0, res, classified = 0, is_timeout = 0; diff --git a/src/afl-fuzz-cmplog.c b/src/afl-fuzz-cmplog.c index 8967d4bc..229aef09 100644 --- a/src/afl-fuzz-cmplog.c +++ b/src/afl-fuzz-cmplog.c @@ -33,15 +33,19 @@ void cmplog_exec_child(afl_forkserver_t *fsrv, char **argv) { setenv("___AFL_EINS_ZWEI_POLIZEI___", "1", 1); - if (fsrv->qemu_mode) { setenv("AFL_DISABLE_LLVM_INSTRUMENTATION", "1", 0); } + if (fsrv->qemu_mode || fsrv->frida_mode || fsrv->cs_mode) { + + setenv("AFL_DISABLE_LLVM_INSTRUMENTATION", "1", 0); + + } if (!fsrv->qemu_mode && !fsrv->frida_mode && argv[0] != fsrv->cmplog_binary) { - argv[0] = fsrv->cmplog_binary; + fsrv->target_path = argv[0] = fsrv->cmplog_binary; } - execv(argv[0], argv); + execv(fsrv->target_path, argv); } diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 1182bd41..c20965b4 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -1120,7 +1120,7 @@ void perform_dry_run(afl_state_t *afl) { } - if (q->var_behavior) { + if (unlikely(q->var_behavior && !afl->afl_env.afl_no_warn_instability)) { WARNF("Instrumentation output varies across runs."); diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 22e5262e..9ea46e7a 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -312,12 +312,18 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { if (notrim) { + if (mutator->afl_custom_init_trim || mutator->afl_custom_trim || + mutator->afl_custom_post_trim) { + + WARNF( + "Custom mutator does not implement all three trim APIs, standard " + "trimming will be used."); + + } + mutator->afl_custom_init_trim = NULL; mutator->afl_custom_trim = NULL; mutator->afl_custom_post_trim = NULL; - ACTF( - "Custom mutator does not implement all three trim APIs, standard " - "trimming will be used."); } @@ -358,6 +364,19 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { } + /* "afl_custom_splice_optout", optional, never called */ + mutator->afl_custom_splice_optout = dlsym(dh, "afl_custom_splice_optout"); + if (!mutator->afl_custom_splice_optout) { + + ACTF("optional symbol 'afl_custom_splice_optout' not found."); + + } else { + + OKF("Found 'afl_custom_splice_optout'."); + afl->custom_splice_optout = 1; + + } + /* "afl_custom_fuzz_send", optional */ mutator->afl_custom_fuzz_send = dlsym(dh, "afl_custom_fuzz_send"); if (!mutator->afl_custom_fuzz_send) { diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 97855607..be526dbd 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -446,9 +446,12 @@ u8 fuzz_one_original(afl_state_t *afl) { ACTF( "Fuzzing test case #%u (%u total, %llu crashes saved, " - "perf_score=%0.0f, exec_us=%llu, hits=%u, map=%u, ascii=%u)...", + "perf_score=%0.0f, weight=%0.0f, favorite=%u, was_fuzzed=%u, " + "exec_us=%llu, hits=%u, map=%u, ascii=%u)...", afl->current_entry, afl->queued_items, afl->saved_crashes, - afl->queue_cur->perf_score, afl->queue_cur->exec_us, + afl->queue_cur->perf_score, afl->queue_cur->weight, + afl->queue_cur->favored, afl->queue_cur->was_fuzzed, + afl->queue_cur->exec_us, likely(afl->n_fuzz) ? afl->n_fuzz[afl->queue_cur->n_fuzz_entry] : 0, afl->queue_cur->bitmap_size, afl->queue_cur->is_ascii); fflush(stdout); @@ -561,11 +564,11 @@ u8 fuzz_one_original(afl_state_t *afl) { } else { - if (afl->cmplog_lvl == 3 || - (afl->cmplog_lvl == 2 && afl->queue_cur->tc_ref) || - afl->queue_cur->favored || - !(afl->fsrv.total_execs % afl->queued_items) || - get_cur_time() - afl->last_find_time > 300000) { // 300 seconds + if (afl->queue_cur->favored || afl->cmplog_lvl == 3 || + (afl->cmplog_lvl == 2 && + (afl->queue_cur->tc_ref || + afl->fsrv.total_execs % afl->queued_items <= 10)) || + get_cur_time() - afl->last_find_time > 250000) { // 250 seconds if (input_to_state_stage(afl, in_buf, out_buf, len)) { @@ -584,7 +587,7 @@ u8 fuzz_one_original(afl_state_t *afl) { if it has gone through deterministic testing in earlier, resumed runs (passed_det). */ - if (likely(afl->queue_cur->passed_det) || likely(afl->skip_deterministic) || + if (likely(afl->skip_deterministic) || likely(afl->queue_cur->passed_det) || likely(perf_score < (afl->queue_cur->depth * 30 <= afl->havoc_max_mult * 100 ? afl->queue_cur->depth * 30 @@ -1908,9 +1911,10 @@ custom_mutator_stage: afl->stage_name = "custom mutator"; afl->stage_short = "custom"; - afl->stage_max = HAVOC_CYCLES * perf_score / afl->havoc_div / 100; afl->stage_val_type = STAGE_VAL_NONE; bool has_custom_fuzz = false; + u32 shift = unlikely(afl->custom_only) ? 7 : 8; + afl->stage_max = (HAVOC_CYCLES * perf_score / afl->havoc_div) >> shift; if (afl->stage_max < HAVOC_MIN) { afl->stage_max = HAVOC_MIN; } @@ -1953,7 +1957,8 @@ custom_mutator_stage: u32 target_len = 0; /* check if splicing makes sense yet (enough entries) */ - if (likely(afl->ready_for_splicing_count > 1)) { + if (likely(!afl->custom_splice_optout && + afl->ready_for_splicing_count > 1)) { /* Pick a random other queue entry for passing to external API that has the necessary length */ @@ -1983,7 +1988,8 @@ custom_mutator_stage: if (unlikely(!mutated_buf)) { - FATAL("Error in custom_fuzz. Size returned: %zu", mutated_size); + // FATAL("Error in custom_fuzz. Size returned: %zu", mutated_size); + break; } @@ -2035,7 +2041,7 @@ custom_mutator_stage: new_hit_cnt = afl->queued_items + afl->saved_crashes; afl->stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt; - afl->stage_cycles[STAGE_CUSTOM_MUTATOR] += afl->stage_max; + afl->stage_cycles[STAGE_CUSTOM_MUTATOR] += afl->stage_cur; #ifdef INTROSPECTION afl->queue_cur->stats_mutated += afl->stage_max; #endif @@ -2063,8 +2069,9 @@ havoc_stage: afl->stage_name = "havoc"; afl->stage_short = "havoc"; - afl->stage_max = (doing_det ? HAVOC_CYCLES_INIT : HAVOC_CYCLES) * - perf_score / afl->havoc_div / 100; + afl->stage_max = ((doing_det ? HAVOC_CYCLES_INIT : HAVOC_CYCLES) * + perf_score / afl->havoc_div) >> + 7; } else { @@ -2073,7 +2080,7 @@ havoc_stage: snprintf(afl->stage_name_buf, STAGE_BUF_SIZE, "splice %u", splice_cycle); afl->stage_name = afl->stage_name_buf; afl->stage_short = "splice"; - afl->stage_max = SPLICE_HAVOC * perf_score / afl->havoc_div / 100; + afl->stage_max = (SPLICE_HAVOC * perf_score / afl->havoc_div) >> 7; } @@ -4621,8 +4628,9 @@ pacemaker_fuzzing: afl->stage_name = MOpt_globals.havoc_stagename; afl->stage_short = MOpt_globals.havoc_stagenameshort; - afl->stage_max = (doing_det ? HAVOC_CYCLES_INIT : HAVOC_CYCLES) * - perf_score / afl->havoc_div / 100; + afl->stage_max = ((doing_det ? HAVOC_CYCLES_INIT : HAVOC_CYCLES) * + perf_score / afl->havoc_div) >> + 7; } else { @@ -4632,7 +4640,7 @@ pacemaker_fuzzing: MOpt_globals.splice_stageformat, splice_cycle); afl->stage_name = afl->stage_name_buf; afl->stage_short = MOpt_globals.splice_stagenameshort; - afl->stage_max = SPLICE_HAVOC * perf_score / afl->havoc_div / 100; + afl->stage_max = (SPLICE_HAVOC * perf_score / afl->havoc_div) >> 7; } @@ -5683,6 +5691,7 @@ pacemaker_fuzzing: } /* block */ + ++afl->queue_cur->fuzz_level; return ret_val; } @@ -5792,13 +5801,11 @@ void pso_updating(afl_state_t *afl) { } -/* larger change for MOpt implementation: the original fuzz_one was renamed - to fuzz_one_original. All documentation references to fuzz_one therefore - mean fuzz_one_original */ - +/* The entry point for the mutator, choosing the default mutator, and/or MOpt + depending on the configuration. */ u8 fuzz_one(afl_state_t *afl) { - int key_val_lv_1 = 0, key_val_lv_2 = 0; + int key_val_lv_1 = -1, key_val_lv_2 = -1; #ifdef _AFL_DOCUMENT_MUTATIONS @@ -5818,7 +5825,12 @@ u8 fuzz_one(afl_state_t *afl) { #endif - // if limit_time_sig == -1 then both are run after each other + /* + -L command line paramter => limit_time_sig value + limit_time_sig == 0 then run the default mutator + limit_time_sig > 0 then run MOpt + limit_time_sig < 0 both are run + */ if (afl->limit_time_sig <= 0) { key_val_lv_1 = fuzz_one_original(afl); } @@ -5840,6 +5852,9 @@ u8 fuzz_one(afl_state_t *afl) { } + if (unlikely(key_val_lv_1 == -1)) { key_val_lv_1 = 0; } + if (likely(key_val_lv_2 == -1)) { key_val_lv_2 = 0; } + return (key_val_lv_1 | key_val_lv_2); } diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index b509b936..2799268b 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -231,8 +231,12 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { PyObject_GetAttrString(py_module, "describe"); py_functions[PY_FUNC_FUZZ_COUNT] = PyObject_GetAttrString(py_module, "fuzz_count"); - if (!py_functions[PY_FUNC_FUZZ]) + if (!py_functions[PY_FUNC_FUZZ]) { + WARNF("fuzz function not found in python module"); + + } + py_functions[PY_FUNC_POST_PROCESS] = PyObject_GetAttrString(py_module, "post_process"); py_functions[PY_FUNC_INIT_TRIM] = @@ -248,6 +252,9 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { PyObject_GetAttrString(py_module, "queue_get"); py_functions[PY_FUNC_FUZZ_SEND] = PyObject_GetAttrString(py_module, "fuzz_send"); + py_functions[PY_FUNC_SPLICE_OPTOUT] = + PyObject_GetAttrString(py_module, "splice_optout"); + if (py_functions[PY_FUNC_SPLICE_OPTOUT]) { afl->custom_splice_optout = 1; } py_functions[PY_FUNC_QUEUE_NEW_ENTRY] = PyObject_GetAttrString(py_module, "queue_new_entry"); py_functions[PY_FUNC_INTROSPECTION] = @@ -394,6 +401,13 @@ void deinit_py(void *py_mutator) { } +void splice_optout_py(void *py_mutator) { + + // this is never called + (void)(py_mutator); + +} + struct custom_mutator *load_custom_mutator_py(afl_state_t *afl, char *module_name) { @@ -474,6 +488,13 @@ struct custom_mutator *load_custom_mutator_py(afl_state_t *afl, } + if (py_functions[PY_FUNC_SPLICE_OPTOUT]) { + + mutator->afl_custom_splice_optout = splice_optout_py; + afl->custom_splice_optout = 1; + + } + if (py_functions[PY_FUNC_QUEUE_NEW_ENTRY]) { mutator->afl_custom_queue_new_entry = queue_new_entry_py; diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index e3faa392..65446799 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -27,6 +27,22 @@ #include <ctype.h> #include <math.h> +#ifdef _STANDALONE_MODULE +void minimize_bits(afl_state_t *afl, u8 *dst, u8 *src) { + + return; + +} + +void run_afl_custom_queue_new_entry(afl_state_t *afl, struct queue_entry *q, + u8 *a, u8 *b) { + + return; + +} + +#endif + /* select next queue entry based on alias algo - fast! */ inline u32 select_next_queue_entry(afl_state_t *afl) { @@ -78,8 +94,8 @@ void create_alias_table(afl_state_t *afl) { afl->alias_probability = (double *)afl_realloc( (void **)&afl->alias_probability, n * sizeof(double)); double *P = (double *)afl_realloc(AFL_BUF_PARAM(out), n * sizeof(double)); - int *S = (u32 *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32)); - int *L = (u32 *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32)); + int *S = (int *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32)); + int *L = (int *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32)); if (!P || !S || !L || !afl->alias_table || !afl->alias_probability) { @@ -247,11 +263,11 @@ void create_alias_table(afl_state_t *afl) { void mark_as_det_done(afl_state_t *afl, struct queue_entry *q) { - u8 fn[PATH_MAX]; - s32 fd; + char fn[PATH_MAX]; + s32 fd; snprintf(fn, PATH_MAX, "%s/queue/.state/deterministic_done/%s", afl->out_dir, - strrchr(q->fname, '/') + 1); + strrchr((char *)q->fname, '/') + 1); fd = open(fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); if (fd < 0) { PFATAL("Unable to create '%s'", fn); } @@ -266,10 +282,10 @@ void mark_as_det_done(afl_state_t *afl, struct queue_entry *q) { void mark_as_variable(afl_state_t *afl, struct queue_entry *q) { - u8 fn[PATH_MAX]; - u8 ldest[PATH_MAX]; + char fn[PATH_MAX]; + char ldest[PATH_MAX]; - u8 *fn_name = strrchr(q->fname, '/') + 1; + char *fn_name = strrchr((char *)q->fname, '/') + 1; sprintf(ldest, "../../%s", fn_name); sprintf(fn, "%s/queue/.state/variable_behavior/%s", afl->out_dir, fn_name); @@ -293,12 +309,12 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) { if (likely(state == q->fs_redundant)) { return; } - u8 fn[PATH_MAX]; + char fn[PATH_MAX]; q->fs_redundant = state; sprintf(fn, "%s/queue/.state/redundant_edges/%s", afl->out_dir, - strrchr(q->fname, '/') + 1); + strrchr((char *)q->fname, '/') + 1); if (state) { @@ -409,7 +425,7 @@ u8 check_if_text_buf(u8 *buf, u32 len) { static u8 check_if_text(afl_state_t *afl, struct queue_entry *q) { - if (q->len < AFL_TXT_MIN_LEN) return 0; + if (q->len < AFL_TXT_MIN_LEN || q->len < AFL_TXT_MAX_LEN) return 0; u8 *buf; int fd; @@ -417,8 +433,8 @@ static u8 check_if_text(afl_state_t *afl, struct queue_entry *q) { ssize_t comp; if (len >= MAX_FILE) len = MAX_FILE - 1; - if ((fd = open(q->fname, O_RDONLY)) < 0) return 0; - buf = afl_realloc(AFL_BUF_PARAM(in_scratch), len + 1); + if ((fd = open((char *)q->fname, O_RDONLY)) < 0) return 0; + buf = (u8 *)afl_realloc(AFL_BUF_PARAM(in_scratch), len + 1); comp = read(fd, buf, len); close(fd); if (comp != (ssize_t)len) return 0; @@ -520,7 +536,8 @@ static u8 check_if_text(afl_state_t *afl, struct queue_entry *q) { void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { - struct queue_entry *q = ck_alloc(sizeof(struct queue_entry)); + struct queue_entry *q = + (struct queue_entry *)ck_alloc(sizeof(struct queue_entry)); q->fname = fname; q->len = len; @@ -554,7 +571,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { afl->cycles_wo_finds = 0; - struct queue_entry **queue_buf = afl_realloc( + struct queue_entry **queue_buf = (struct queue_entry **)afl_realloc( AFL_BUF_PARAM(queue), afl->queued_items * sizeof(struct queue_entry *)); if (unlikely(!queue_buf)) { PFATAL("alloc"); } queue_buf[afl->queued_items - 1] = q; @@ -574,7 +591,11 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { } /* only redqueen currently uses is_ascii */ - if (afl->shm.cmplog_mode) q->is_ascii = check_if_text(afl, q); + if (unlikely(afl->shm.cmplog_mode && !q->is_ascii)) { + + q->is_ascii = check_if_text(afl, q); + + } } @@ -704,7 +725,7 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { if (!q->trace_mini) { u32 len = (afl->fsrv.map_size >> 3); - q->trace_mini = ck_alloc(len); + q->trace_mini = (u8 *)ck_alloc(len); minimize_bits(afl, q->trace_mini, afl->fsrv.trace_bits); } @@ -1007,10 +1028,16 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { break; case LIN: + // Don't modify perf_score for unfuzzed seeds + if (!q->fuzz_level) break; + factor = q->fuzz_level / (afl->n_fuzz[q->n_fuzz_entry] + 1); break; case QUAD: + // Don't modify perf_score for unfuzzed seeds + if (!q->fuzz_level) break; + factor = q->fuzz_level * q->fuzz_level / (afl->n_fuzz[q->n_fuzz_entry] + 1); break; @@ -1090,19 +1117,19 @@ inline void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, if (len != old_len) { afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len; - q->testcase_buf = realloc(q->testcase_buf, len); + q->testcase_buf = (u8 *)realloc(q->testcase_buf, len); if (unlikely(!q->testcase_buf)) { - PFATAL("Unable to malloc '%s' with len %u", q->fname, len); + PFATAL("Unable to malloc '%s' with len %u", (char *)q->fname, len); } } - int fd = open(q->fname, O_RDONLY); + int fd = open((char *)q->fname, O_RDONLY); - if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } + if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", (char *)q->fname); } ck_read(fd, q->testcase_buf, len, q->fname); close(fd); @@ -1122,7 +1149,7 @@ inline void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q, if (likely(len != old_len)) { - u8 *ptr = realloc(q->testcase_buf, len); + u8 *ptr = (u8 *)realloc(q->testcase_buf, len); if (likely(ptr)) { @@ -1154,23 +1181,23 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { if (unlikely(q == afl->queue_cur)) { - buf = afl_realloc((void **)&afl->testcase_buf, len); + buf = (u8 *)afl_realloc((void **)&afl->testcase_buf, len); } else { - buf = afl_realloc((void **)&afl->splicecase_buf, len); + buf = (u8 *)afl_realloc((void **)&afl->splicecase_buf, len); } if (unlikely(!buf)) { - PFATAL("Unable to malloc '%s' with len %u", q->fname, len); + PFATAL("Unable to malloc '%s' with len %u", (char *)q->fname, len); } - int fd = open(q->fname, O_RDONLY); + int fd = open((char *)q->fname, O_RDONLY); - if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } + if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", (char *)q->fname); } ck_read(fd, buf, len, q->fname); close(fd); @@ -1214,7 +1241,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { do_once = 1; // release unneeded memory - afl->q_testcase_cache = ck_realloc( + afl->q_testcase_cache = (struct queue_entry **)ck_realloc( afl->q_testcase_cache, (afl->q_testcase_max_cache_entries + 1) * sizeof(size_t)); @@ -1261,15 +1288,15 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { /* Map the test case into memory. */ - int fd = open(q->fname, O_RDONLY); + int fd = open((char *)q->fname, O_RDONLY); - if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } + if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", (char *)q->fname); } - q->testcase_buf = malloc(len); + q->testcase_buf = (u8 *)malloc(len); if (unlikely(!q->testcase_buf)) { - PFATAL("Unable to malloc '%s' with len %u", q->fname, len); + PFATAL("Unable to malloc '%s' with len %u", (char *)q->fname, len); } @@ -1332,11 +1359,11 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q, /* Map the test case into memory. */ - q->testcase_buf = malloc(len); + q->testcase_buf = (u8 *)malloc(len); if (unlikely(!q->testcase_buf)) { - PFATAL("Unable to malloc '%s' with len %u", q->fname, len); + PFATAL("Unable to malloc '%s' with len %u", (char *)q->fname, len); } diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 8da1df13..6e4a655b 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -1035,7 +1035,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } else { - diff = 0; + o_diff = 0; } @@ -1624,6 +1624,8 @@ static void try_to_add_to_dictN(afl_state_t *afl, u128 v, u8 size) { } + if (cons_0 > 1 || cons_ff > 1) { return; } + } maybe_add_auto(afl, (u8 *)&v + off, size); diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 7dd83150..f5425011 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -523,7 +523,7 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, } - if (unlikely(!var_detected)) { + if (unlikely(!var_detected && !afl->afl_env.afl_no_warn_instability)) { // note: from_queue seems to only be set during initialization if (afl->afl_env.afl_no_ui || from_queue) { diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 896b5f71..6d8c8758 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -204,6 +204,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_no_affinity = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_NO_WARN_INSTABILITY", + + afl_environment_variable_len)) { + + afl->afl_env.afl_no_warn_instability = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_TRY_AFFINITY", afl_environment_variable_len)) { @@ -292,6 +299,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_ignore_problems = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_IGNORE_TIMEOUTS", + + afl_environment_variable_len)) { + + afl->afl_env.afl_ignore_timeouts = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", afl_environment_variable_len)) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index b8114a7f..4914ce0b 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -258,8 +258,9 @@ static void usage(u8 *argv0, int more_help) { "AFL_FORKSRV_INIT_TMOUT: time spent waiting for forkserver during startup (in ms)\n" "AFL_HANG_TMOUT: override timeout value (in milliseconds)\n" "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: don't warn about core dump handlers\n" - "AFL_IGNORE_UNKNOWN_ENVS: don't warn on unknown env vars\n" "AFL_IGNORE_PROBLEMS: do not abort fuzzing if an incorrect setup is detected\n" + "AFL_IGNORE_TIMEOUTS: do not process or save any timeouts\n" + "AFL_IGNORE_UNKNOWN_ENVS: don't warn on unknown env vars\n" "AFL_IMPORT_FIRST: sync and import test cases from other fuzzer instances first\n" "AFL_INPUT_LEN_MIN/AFL_INPUT_LEN_MAX: like -g/-G set min/max fuzz length produced\n" "AFL_PIZZA_MODE: 1 - enforce pizza mode, 0 - disable for April 1st\n" @@ -1297,6 +1298,13 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->is_main_node == 1 && afl->schedule != FAST && + afl->schedule != EXPLORE) { + + FATAL("-M is compatible only with fast and explore -p power schedules"); + + } + if (optind == argc || !afl->in_dir || !afl->out_dir || show_help) { usage(argv[0], show_help); @@ -1579,6 +1587,29 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->limit_time_sig > 0 && afl->custom_mutators_count) { + + if (afl->custom_only) { + + FATAL("Custom mutators are incompatible with MOpt (-L)"); + + } + + u32 custom_fuzz = 0; + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_fuzz) { custom_fuzz = 1; } + + }); + + if (custom_fuzz) { + + WARNF("afl_custom_fuzz is incompatible with MOpt (-L)"); + + } + + } + if (afl->afl_env.afl_max_det_extras) { s32 max_det_extras = atoi(afl->afl_env.afl_max_det_extras); @@ -2081,6 +2112,7 @@ int main(int argc, char **argv_orig, char **envp) { afl->cmplog_fsrv.qemu_mode = afl->fsrv.qemu_mode; afl->cmplog_fsrv.frida_mode = afl->fsrv.frida_mode; afl->cmplog_fsrv.cmplog_binary = afl->cmplog_binary; + afl->cmplog_fsrv.target_path = afl->fsrv.target_path; afl->cmplog_fsrv.init_child_func = cmplog_exec_child; if ((map_size <= DEFAULT_SHMEM_SIZE || @@ -2549,6 +2581,7 @@ int main(int argc, char **argv_orig, char **envp) { skipped_fuzz = fuzz_one(afl); #ifdef INTROSPECTION ++afl->queue_cur->stats_selected; + if (unlikely(skipped_fuzz)) { ++afl->queue_cur->stats_skipped; diff --git a/src/afl-gotcpu.c b/src/afl-gotcpu.c index c5b8a27a..4f851099 100644 --- a/src/afl-gotcpu.c +++ b/src/afl-gotcpu.c @@ -92,7 +92,7 @@ static u32 measure_preemption(u32 target_ms) { volatile u32 v1, v2 = 0; u64 st_t, en_t, st_c, en_c, real_delta, slice_delta; - s32 loop_repeats = 0; + // s32 loop_repeats = 0; st_t = get_cur_time_us(); st_c = get_cpu_usage_us(); @@ -113,7 +113,7 @@ repeat_loop: if (en_t - st_t < target_ms * 1000) { - loop_repeats++; + // loop_repeats++; goto repeat_loop; } @@ -214,7 +214,13 @@ int main(int argc, char **argv) { #if defined(__linux__) if (sched_setaffinity(0, sizeof(c), &c)) { - PFATAL("sched_setaffinity failed for cpu %d", i); + const char *error_code = "Unkown error code"; + if (errno == EFAULT) error_code = "EFAULT"; + if (errno == EINVAL) error_code = "EINVAL"; + if (errno == EPERM) error_code = "EPERM"; + if (errno == ESRCH) error_code = "ESRCH"; + + PFATAL("sched_setaffinity failed for cpu %d, error: %s", i, error_code); } diff --git a/test/test-llvm.sh b/test/test-llvm.sh index ce64d76c..52be04ef 100755 --- a/test/test-llvm.sh +++ b/test/test-llvm.sh @@ -257,12 +257,13 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { } rm -f test-compcov test.out instrumentlist.txt AFL_LLVM_CMPLOG=1 ../afl-clang-fast -o test-cmplog test-cmplog.c > /dev/null 2>&1 + ../afl-clang-fast -o test-c test-cmplog.c > /dev/null 2>&1 test -e test-cmplog && { $ECHO "$GREY[*] running afl-fuzz for llvm_mode cmplog, this will take approx 10 seconds" { mkdir -p in echo 00000000000000000000000000000000 > in/in - AFL_BENCH_UNTIL_CRASH=1 ../afl-fuzz -m none -V60 -i in -o out -c./test-cmplog -- ./test-cmplog >>errors 2>&1 + AFL_BENCH_UNTIL_CRASH=1 ../afl-fuzz -m none -V60 -i in -o out -c./test-cmplog -- ./test-c >>errors 2>&1 } >>errors 2>&1 test -n "$( ls out/default/crashes/id:000000* out/default/hangs/id:000000* 2>/dev/null )" & { $ECHO "$GREEN[+] afl-fuzz is working correctly with llvm_mode cmplog" @@ -277,7 +278,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { $ECHO "$YELLOW[-] we cannot test llvm_mode cmplog because it is not present" INCOMPLETE=1 } - rm -rf errors test-cmplog in core.* + rm -rf errors test-cmplog test-c in core.* ../afl-clang-fast -o test-persistent ../utils/persistent_mode/persistent_demo.c > /dev/null 2>&1 test -e test-persistent && { echo foo | AFL_QUIET=1 ../afl-showmap -m ${MEM_LIMIT} -o /dev/null -q -r ./test-persistent && { |