From 0f72a7473f994dad89340cca76e62545829ef493 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Tue, 27 Sep 2022 13:45:39 +0000 Subject: Fix docs for fuzz_count in python --- docs/custom_mutators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docs/custom_mutators.md') diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index 6f3353ec..6b72430a 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -68,7 +68,7 @@ Python: def init(seed): pass -def fuzz_count(buf, add_buf, max_size): +def fuzz_count(buf): return cnt def fuzz(buf, add_buf, max_size): -- cgit 1.4.1 From c058ce37bbd578be91b8c5d26161ecb281568725 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 1 Nov 2022 14:05:31 +0100 Subject: remove python2 reference --- GNUmakefile | 2 +- docs/custom_mutators.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'docs/custom_mutators.md') diff --git a/GNUmakefile b/GNUmakefile index d1f28a76..f5f2dcb2 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -425,7 +425,7 @@ test_python: @echo "[+] $(PYTHON_VERSION) support seems to be working." else test_python: - @echo "[-] You seem to need to install the package python3-dev, python2-dev or python-dev (and perhaps python[23]-apt), but it is optional so we continue" + @echo "[-] You seem to need to install the package python3-dev or python-dev (and perhaps python[3]-apt), but it is optional so we continue" endif .PHONY: ready diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index 6b72430a..ffd3cce8 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -269,10 +269,10 @@ sudo apt install python-dev ``` Then, AFL++ can be compiled with Python support. The AFL++ Makefile detects -Python 2 and 3 through `python-config` if it is in the PATH and compiles -`afl-fuzz` with the feature if available. +Python3 through `python-config`/`python3-config` if it is in the PATH and +compiles `afl-fuzz` with the feature if available. -Note: for some distributions, you might also need the package `python[23]-apt`. +Note: for some distributions, you might also need the package `python[3]-apt`. In case your setup is different, set the necessary variables like this: `PYTHON_INCLUDE=/path/to/python/include LDFLAGS=-L/path/to/python/lib make`. -- cgit 1.4.1 From c5f8869778b237f89cc4c12c1961a9ef559179b8 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 11 Nov 2022 10:46:45 +0100 Subject: afl_custom_fuzz_send added --- docs/Changelog.md | 8 ++++++-- docs/custom_mutators.md | 10 ++++++++++ include/afl-fuzz.h | 13 +++++++++++++ src/afl-fuzz-mutators.c | 8 ++++++++ src/afl-fuzz-python.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ src/afl-fuzz-run.c | 42 +++++++++++++++++++++++++++++++++++++----- utils/afl_proxy/README.md | 5 +++++ 7 files changed, 124 insertions(+), 7 deletions(-) (limited to 'docs/custom_mutators.md') diff --git a/docs/Changelog.md b/docs/Changelog.md index 38e2e6bc..b975fa76 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -4,8 +4,12 @@ release of the tool. See README.md for the general instruction manual. ### Version ++4.05a (dev) - - your PR? :) - + - afl-fuzz: + - added afl_custom_fuzz_send custom mutator feature. Now your can + send fuzz data to the target as you need, e.g. via IPC. + - afl-showmap/afl-cmin + - -t none now translates to -t 120000 (120 seconds) + - unicorn_mode updated ### Version ++4.04c (release) - fix gramatron and grammar_mutator build scripts diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index ffd3cce8..ecc004e2 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -57,6 +57,7 @@ int afl_custom_post_trim(void *data, unsigned char success); size_t afl_custom_havoc_mutation(void *data, unsigned char *buf, size_t buf_size, unsigned char **out_buf, size_t max_size); unsigned char afl_custom_havoc_mutation_probability(void *data); unsigned char afl_custom_queue_get(void *data, const unsigned char *filename); +void (*afl_custom_fuzz_send)(void *data, const u8 *buf, size_t buf_size); u8 afl_custom_queue_new_entry(void *data, const unsigned char *filename_new_queue, const unsigned int *filename_orig_queue); const char* afl_custom_introspection(my_mutator_t *data); void afl_custom_deinit(void *data); @@ -98,6 +99,9 @@ def havoc_mutation_probability(): def queue_get(filename): return True +def fuzz_send(buf): + pass + def queue_new_entry(filename_new_queue, filename_orig_queue): return False @@ -168,6 +172,12 @@ def deinit(): # optional for Python to the target, e.g. if it is too short, too corrupted, etc. If so, return a NULL buffer and zero length (or a 0 length string in Python). +- `fuzz_send` (optional): + + This method can be used if you want to send data to the target yourself, + e.g. via IPC. This replaces some usage of utils/afl_proxy but requires + that you start the target with afl-fuzz. + - `queue_new_entry` (optional): This methods is called after adding a new test case to the queue. If the diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index c8ca8e9b..a63b7493 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -968,6 +968,19 @@ struct custom_mutator { */ u8 (*afl_custom_queue_get)(void *data, const u8 *filename); + /** + * This method can be used if you want to send data to the target yourself, + * e.g. via IPC. This replaces some usage of utils/afl_proxy but requires + * that you start the target with afl-fuzz. + * + * (Optional) + * + * @param data pointer returned in afl_custom_init by this custom mutator + * @param buf Buffer containing the test case + * @param buf_size Size of the test case + */ + void (*afl_custom_fuzz_send)(void *data, const u8 *buf, size_t buf_size); + /** * Allow for additional analysis (e.g. calling a different tool that does a * different kind of coverage and saves this for the custom mutator). diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index b9daebfa..f3d0bc96 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -314,6 +314,14 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { } + /* "afl_custom_fuzz_send", optional */ + mutator->afl_custom_fuzz_send = dlsym(dh, "afl_custom_fuzz_send"); + if (!mutator->afl_custom_fuzz_send) { + + ACTF("optional symbol 'afl_custom_fuzz_send' not found."); + + } + /* "afl_custom_queue_new_entry", optional */ mutator->afl_custom_queue_new_entry = dlsym(dh, "afl_custom_queue_new_entry"); if (!mutator->afl_custom_queue_new_entry) { diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index a43d80bb..54b05978 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -246,6 +246,8 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { PyObject_GetAttrString(py_module, "havoc_mutation_probability"); py_functions[PY_FUNC_QUEUE_GET] = PyObject_GetAttrString(py_module, "queue_get"); + py_functions[PY_FUNC_FUZZ_SEND] = + PyObject_GetAttrString(py_module, "fuzz_send"); py_functions[PY_FUNC_QUEUE_NEW_ENTRY] = PyObject_GetAttrString(py_module, "queue_new_entry"); py_functions[PY_FUNC_INTROSPECTION] = @@ -466,6 +468,12 @@ struct custom_mutator *load_custom_mutator_py(afl_state_t *afl, } + if (py_functions[PY_FUNC_FUZZ_SEND]) { + + mutator->afl_custom_fuzz_send = fuzz_send_py; + + } + if (py_functions[PY_FUNC_QUEUE_NEW_ENTRY]) { mutator->afl_custom_queue_new_entry = queue_new_entry_py; @@ -893,6 +901,43 @@ u8 queue_get_py(void *py_mutator, const u8 *filename) { } +void fuzz_send_py(void *py_mutator, const u8 *buf, size_t buf_size) { + + PyObject *py_args, *py_value; + + py_args = PyTuple_New(1); + py_value = PyByteArray_FromStringAndSize(buf, buf_size); + if (!py_value) { + + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + + } + + PyTuple_SetItem(py_args, 0, py_value); + + py_value = PyObject_CallObject( + ((py_mutator_t *)py_mutator)->py_functions[PY_FUNC_FUZZ_SEND], py_args); + Py_DECREF(py_args); + + if (py_value != NULL) { + + #if PY_MAJOR_VERSION >= 3 + u32 retcnt = (u32)PyLong_AsLong(py_value); + #else + u32 retcnt = PyInt_AsLong(py_value); + #endif + Py_DECREF(py_value); + + } else { + + PyErr_Print(); + FATAL("Call failed"); + + } + +} + u8 queue_new_entry_py(void *py_mutator, const u8 *filename_new_queue, const u8 *filename_orig_queue) { diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index ee4a3298..a680228d 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -133,9 +133,25 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { if (new_mem != *mem) { *mem = new_mem; } - /* everything as planned. use the potentially new data. */ - afl_fsrv_write_to_testcase(&afl->fsrv, *mem, new_size); - len = new_size; + if (unlikely(afl->custom_mutators_count)) { + + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_fuzz_send) { + + el->afl_custom_fuzz_send(el->data, *mem, new_size); + + } + + }); + + } else { + + /* everything as planned. use the potentially new data. */ + afl_fsrv_write_to_testcase(&afl->fsrv, *mem, new_size); + len = new_size; + + } } else { @@ -149,8 +165,24 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { } - /* boring uncustom. */ - afl_fsrv_write_to_testcase(&afl->fsrv, *mem, len); + if (unlikely(afl->custom_mutators_count)) { + + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_fuzz_send) { + + el->afl_custom_fuzz_send(el->data, *mem, len); + + } + + }); + + } else { + + /* boring uncustom. */ + afl_fsrv_write_to_testcase(&afl->fsrv, *mem, len); + + } } diff --git a/utils/afl_proxy/README.md b/utils/afl_proxy/README.md index 3c768a19..7965659d 100644 --- a/utils/afl_proxy/README.md +++ b/utils/afl_proxy/README.md @@ -7,3 +7,8 @@ You only need to change the while() loop of the main() to send the data of buf[] with length len to the target and write the coverage information to __afl_area_ptr[__afl_map_size] +UPDATE: you can also use [custom mutators](../../docs/custom_mutators.md) with +afl_custom_fuzz_send to send data to a target, which is much more efficient! +But you can only use this feature if you start the target via afl-fuzz and +a forkserver is active (e.g. via -Q qemu_mode or source compiled). + -- cgit 1.4.1 From 4a7cd53f64abf4eac311621220afc927574fdcea Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 14 Nov 2022 21:26:17 +0100 Subject: custom_send example --- custom_mutators/examples/custom_send.c | 56 ++++++++++++++++++++++++++++++++ docs/custom_mutators.md | 1 + src/afl-fuzz-mutators.c | 58 +++++++++++++++++++++++++++++++++- 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 custom_mutators/examples/custom_send.c (limited to 'docs/custom_mutators.md') diff --git a/custom_mutators/examples/custom_send.c b/custom_mutators/examples/custom_send.c new file mode 100644 index 00000000..ffea927e --- /dev/null +++ b/custom_mutators/examples/custom_send.c @@ -0,0 +1,56 @@ +// cc -O3 -fPIC -shared -g -o custom_send.so -I../../include custom_send.c +// cd ../.. +// afl-cc -o test-instr test-instr.c +// afl-fuzz -i in -o out -- ./test-instr -f /tmp/foo + +#include "custom_mutator_helpers.h" + +#include +#include +#include +#include +#include + +typedef struct my_mutator { + + afl_t *afl; + +} my_mutator_t; + +my_mutator_t *afl_custom_init(afl_t *afl, unsigned int seed) { + + my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + data->afl = afl; + + return data; + +} + +void afl_custom_fuzz_send(my_mutator_t *data, uint8_t *buf, size_t buf_size) { + + int fd = open("/tmp/foo", O_CREAT | O_NOFOLLOW | O_TRUNC | O_RDWR, 0644); + + if (fd >= 0) { + + (void)write(fd, buf, buf_size); + close(fd); + + } + + return; + +} + +void afl_custom_deinit(my_mutator_t *data) { + + free(data); + +} + diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index ecc004e2..4ffeda7a 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -177,6 +177,7 @@ def deinit(): # optional for Python This method can be used if you want to send data to the target yourself, e.g. via IPC. This replaces some usage of utils/afl_proxy but requires that you start the target with afl-fuzz. + Example: [custom_mutators/examples/custom_send.c](custom_mutators/examples/custom_send.c) - `queue_new_entry` (optional): diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index f3d0bc96..ef30b993 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -211,8 +211,16 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { WARNF("Symbol 'afl_custom_mutator' not found."); + } else { + + OKF("Found 'afl_custom_mutator'."); + } + } else { + + OKF("Found 'afl_custom_mutator'."); + } /* "afl_custom_introspection", optional */ @@ -222,6 +230,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { ACTF("optional symbol 'afl_custom_introspection' not found."); + } else { + + OKF("Found 'afl_custom_introspection'."); + } #endif @@ -232,6 +244,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { ACTF("optional symbol 'afl_custom_fuzz_count' not found."); + } else { + + OKF("Found 'afl_custom_fuzz_count'."); + } /* "afl_custom_deinit", optional for backward compatibility */ @@ -248,6 +264,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { ACTF("optional symbol 'afl_custom_post_process' not found."); + } else { + + OKF("Found 'afl_custom_post_process'."); + } u8 notrim = 0; @@ -258,6 +278,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { notrim = 1; ACTF("optional symbol 'afl_custom_init_trim' not found."); + } else { + + OKF("Found 'afl_custom_init_trim'."); + } /* "afl_custom_trim", optional */ @@ -267,6 +291,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { notrim = 1; ACTF("optional symbol 'afl_custom_trim' not found."); + } else { + + OKF("Found 'afl_custom_trim'."); + } /* "afl_custom_post_trim", optional */ @@ -276,6 +304,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { notrim = 1; ACTF("optional symbol 'afl_custom_post_trim' not found."); + } else { + + OKF("Found 'afl_custom_post_trim'."); + } if (notrim) { @@ -295,6 +327,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { ACTF("optional symbol 'afl_custom_havoc_mutation' not found."); + } else { + + OKF("Found 'afl_custom_havoc_mutation'."); + } /* "afl_custom_havoc_mutation", optional */ @@ -304,6 +340,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { ACTF("optional symbol 'afl_custom_havoc_mutation_probability' not found."); + } else { + + OKF("Found 'afl_custom_havoc_mutation_probability'."); + } /* "afl_custom_queue_get", optional */ @@ -312,6 +352,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { ACTF("optional symbol 'afl_custom_queue_get' not found."); + } else { + + OKF("Found 'afl_custom_queue_get'."); + } /* "afl_custom_fuzz_send", optional */ @@ -320,6 +364,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { ACTF("optional symbol 'afl_custom_fuzz_send' not found."); + } else { + + OKF("Found 'afl_custom_fuzz_send'."); + } /* "afl_custom_queue_new_entry", optional */ @@ -328,13 +376,21 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { ACTF("optional symbol 'afl_custom_queue_new_entry' not found"); + } else { + + OKF("Found 'afl_custom_queue_new_entry'."); + } /* "afl_custom_describe", optional */ mutator->afl_custom_describe = dlsym(dh, "afl_custom_describe"); if (!mutator->afl_custom_describe) { - ACTF("Symbol 'afl_custom_describe' not found."); + ACTF("optional symbol 'afl_custom_describe' not found."); + + } else { + + OKF("Found 'afl_custom_describe'."); } -- cgit 1.4.1 From 14d8eb9e40a6329abcb2f153174b543349c68c13 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 18 Jan 2023 22:17:14 +0100 Subject: autotoken: splicing; splice_optout --- custom_mutators/autotokens/Makefile | 6 +- custom_mutators/autotokens/autotokens.cpp | 103 +++++++++++++++++++++++++++--- docs/custom_mutators.md | 11 ++++ include/afl-fuzz.h | 14 ++++ src/afl-fuzz-mutators.c | 13 ++++ src/afl-fuzz-one.c | 3 +- src/afl-fuzz-python.c | 16 +++++ 7 files changed, 155 insertions(+), 11 deletions(-) (limited to 'docs/custom_mutators.md') diff --git a/custom_mutators/autotokens/Makefile b/custom_mutators/autotokens/Makefile index 8af63635..ab1da4b6 100644 --- a/custom_mutators/autotokens/Makefile +++ b/custom_mutators/autotokens/Makefile @@ -1,5 +1,9 @@ ifdef debug - CFLAGS += "-fsanitize=address -Wall" + CFLAGS += -fsanitize=address -Wall + CXX := clang++ +endif +ifdef DEBUG + CFLAGS += -fsanitize=address -Wall CXX := clang++ endif diff --git a/custom_mutators/autotokens/autotokens.cpp b/custom_mutators/autotokens/autotokens.cpp index 7aecb010..c9ec4352 100644 --- a/custom_mutators/autotokens/autotokens.cpp +++ b/custom_mutators/autotokens/autotokens.cpp @@ -19,6 +19,13 @@ extern "C" { #define AUTOTOKENS_ALTERNATIVE_TOKENIZE 0 #define AUTOTOKENS_CHANGE_MIN 8 #define AUTOTOKENS_WHITESPACE " " +#define AUTOTOKENS_SIZE_MIN 8 +#define AUTOTOKENS_SPLICE_MIN 4 +#define AUTOTOKENS_SPLICE_MAX 64 + +#if AUTOTOKENS_SPLICE_MIN >= AUTOTOKENS_SIZE_MIN + #error SPLICE_MIN must be lower than SIZE_MIN +#endif using namespace std; @@ -42,6 +49,7 @@ static u32 extras_cnt, a_extras_cnt; static u64 all_spaces, all_tabs, all_lf, all_ws; static u64 all_structure_items; static unordered_map *> file_mapping; +static unordered_map *> id_mapping; static unordered_map token_to_id; static unordered_map id_to_token; static string whitespace = AUTOTOKENS_WHITESPACE; @@ -76,6 +84,8 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, u8 **out_buf, u8 *add_buf, size_t add_buf_size, size_t max_size) { + (void)(data); + if (s == NULL) { *out_buf = NULL; @@ -92,14 +102,14 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, afl_ptr->havoc_div / 256)); // DEBUG(stderr, "structure size: %lu, rounds: %u \n", m.size(), rounds); - u32 max_rand = 7; + u32 max_rand = 14; for (i = 0; i < rounds; ++i) { switch (rand_below(afl_ptr, max_rand)) { /* CHANGE */ - case 0 ... 3: /* fall through */ + case 0 ... 7: /* fall through */ { u32 pos = rand_below(afl_ptr, m_size); @@ -122,18 +132,19 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, } /* INSERT (m_size +1 so we insert also after last place) */ - case 4 ... 5: { + case 8 ... 9: { u32 new_item; do { new_item = rand_below(afl_ptr, current_id); - } while (!alternative_tokenize && new_item >= whitespace_ids); + } while (unlikely(!alternative_tokenize && new_item >= whitespace_ids)); u32 pos = rand_below(afl_ptr, m_size + 1); m.insert(m.begin() + pos, new_item); ++m_size; + DEBUG(stderr, "INS: %u at %u\n", new_item, pos); if (likely(!alternative_tokenize)) { @@ -168,8 +179,63 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, } + /* SPLICING */ + case 10 ... 11: { + + u32 strategy = rand_below(afl_ptr, 4), dst_off, n; + auto src = id_mapping[rand_below(afl_ptr, valid_structures)]; + u32 src_size = src->size(); + u32 src_off = rand_below(afl_ptr, src_size - AUTOTOKENS_SPLICE_MIN); + u32 rand_r = 1 + MAX(AUTOTOKENS_SPLICE_MIN, + MIN(AUTOTOKENS_SPLICE_MAX, src_size - src_off)); + + switch (strategy) { + + // insert + case 0: { + + dst_off = rand_below(afl_ptr, m_size); + n = AUTOTOKENS_SPLICE_MIN + + rand_below(afl_ptr, MIN(AUTOTOKENS_SPLICE_MAX, + rand_r - AUTOTOKENS_SPLICE_MIN)); + m.insert(m.begin() + dst_off, src->begin() + src_off, + src->begin() + src_off + n); + m_size += n; + DEBUG(stderr, "SPLICE-INS: %u at %u\n", n, dst_off); + break; + + } + + // overwrite + default: { + + dst_off = rand_below(afl_ptr, m_size - AUTOTOKENS_SPLICE_MIN); + n = AUTOTOKENS_SPLICE_MIN + + rand_below( + afl_ptr, + MIN(AUTOTOKENS_SPLICE_MAX - AUTOTOKENS_SPLICE_MIN, + MIN(m_size - dst_off - AUTOTOKENS_SPLICE_MIN, + src_size - src_off - AUTOTOKENS_SPLICE_MIN))); + + for (u32 i = 0; i < n; ++i) { + + m[dst_off + i] = (*src)[src_off + i]; + + } + + DEBUG(stderr, "SPLICE-MUT: %u at %u\n", n, dst_off); + break; + + } + + } + + break; + + } + /* ERASE - only if large enough */ - case 6: { + case 12 ... 13: { if (m_size > 8) { @@ -178,7 +244,7 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, } else { - max_rand = 6; + max_rand = 12; } @@ -236,12 +302,15 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, extern "C" unsigned char afl_custom_queue_get(void *data, const unsigned char *filename) { + (void)(data); + if (likely(!debug)) { if ((afl_ptr->shm.cmplog_mode && !afl_ptr->queue_cur->is_ascii) || (only_fav && !afl_ptr->queue_cur->favored)) { s = NULL; + DEBUG(stderr, "cmplog not ascii or only_fav and not favorite\n"); return 0; } @@ -334,8 +403,8 @@ extern "C" unsigned char afl_custom_queue_get(void *data, fclose(fp); file_mapping[fn] = structure; // NULL ptr so we don't read the file again - DEBUG(stderr, "Too short (%lu) %s\n", len, filename); s = NULL; + DEBUG(stderr, "Too short (%lu) %s\n", len, filename); return 0; } @@ -362,8 +431,8 @@ extern "C" unsigned char afl_custom_queue_get(void *data, if (((len * AFL_TXT_MIN_PERCENT) / 100) > valid_chars) { file_mapping[fn] = NULL; - DEBUG(stderr, "Not text (%lu) %s\n", len, filename); s = NULL; + DEBUG(stderr, "Not text (%lu) %s\n", len, filename); return 0; } @@ -766,6 +835,15 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } + if (tokens.size() < AUTOTOKENS_SIZE_MIN) { + + file_mapping[fn] = NULL; + s = NULL; + DEBUG(stderr, "too few tokens\n"); + return 0; + + } + /* Now we transform the tokens into an ID list and saved that */ structure = new vector(); @@ -791,8 +869,9 @@ extern "C" unsigned char afl_custom_queue_get(void *data, // save the token structure to the file mapping file_mapping[fn] = structure; - s = structure; + id_mapping[valid_structures] = structure; ++valid_structures; + s = structure; all_structure_items += structure->size(); // we are done! @@ -897,6 +976,12 @@ extern "C" my_mutator_t *afl_custom_init(afl_state *afl, unsigned int seed) { } +extern "C" void afl_custom_splice_optout(my_mutator_t *data) { + + (void)(data); + +} + extern "C" void afl_custom_deinit(my_mutator_t *data) { /* we use this to print statistics at exit :-) diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index 4ffeda7a..322caa5b 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -48,6 +48,7 @@ C/C++: ```c void *afl_custom_init(afl_state_t *afl, unsigned int seed); unsigned int afl_custom_fuzz_count(void *data, const unsigned char *buf, size_t buf_size); +void afl_custom_splice_optout(void *data); size_t afl_custom_fuzz(void *data, unsigned char *buf, size_t buf_size, unsigned char **out_buf, unsigned char *add_buf, size_t add_buf_size, size_t max_size); const char *afl_custom_describe(void *data, size_t max_description_len); size_t afl_custom_post_process(void *data, unsigned char *buf, size_t buf_size, unsigned char **out_buf); @@ -72,6 +73,9 @@ def init(seed): def fuzz_count(buf): return cnt +def splice_optout() + pass + def fuzz(buf, add_buf, max_size): return mutated_out @@ -132,6 +136,13 @@ def deinit(): # optional for Python for a specific queue entry, use this function. This function is most useful if `AFL_CUSTOM_MUTATOR_ONLY` is **not** used. +- `splice_optout` (optional): + + If this function is present, no splicing target is passed to the `fuzz` + function. This saves time if splicing data is not needed by the custom + fuzzing function. + This function is never called, just needs to be present to activate. + - `fuzz` (optional): This method performs custom mutations on a given input. It also accepts an diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 69fea579..1e8d085d 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -344,6 +344,7 @@ enum { /* 12 */ PY_FUNC_INTROSPECTION, /* 13 */ PY_FUNC_DESCRIBE, /* 14 */ PY_FUNC_FUZZ_SEND, + /* 15 */ PY_FUNC_SPLICE_OPTOUT, PY_FUNC_COUNT }; @@ -495,6 +496,7 @@ typedef struct afl_state { no_unlink, /* do not unlink cur_input */ debug, /* Debug mode */ custom_only, /* Custom mutator only mode */ + custom_splice_optout, /* Custom mutator no splice buffer */ is_main_node, /* if this is the main node */ is_secondary_node, /* if this is a secondary instance */ pizza_is_served; /* pizza mode */ @@ -828,6 +830,17 @@ struct custom_mutator { */ u32 (*afl_custom_fuzz_count)(void *data, const u8 *buf, size_t buf_size); + /** + * Opt-out of a splicing input for the fuzz mutator + * + * Empty dummy function. It's presence tells afl-fuzz not to pass a + * splice data pointer and len. + * + * @param data pointer returned in afl_custom_init by this custom mutator + * @noreturn + */ + void (*afl_custom_splice_optout)(void *data); + /** * Perform custom mutations on a given input * @@ -1057,6 +1070,7 @@ u8 havoc_mutation_probability_py(void *); u8 queue_get_py(void *, const u8 *); const char *introspection_py(void *); u8 queue_new_entry_py(void *, const u8 *, const u8 *); +void splice_optout(void *); void deinit_py(void *); #endif diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 22e5262e..ce43064a 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -358,6 +358,19 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { } + /* "afl_custom_splice_optout", optional, never called */ + mutator->afl_custom_splice_optout = dlsym(dh, "afl_custom_splice_optout"); + if (!mutator->afl_custom_splice_optout) { + + ACTF("optional symbol 'afl_custom_splice_optout' not found."); + + } else { + + OKF("Found 'afl_custom_splice_optout'."); + afl->custom_splice_optout = 1; + + } + /* "afl_custom_fuzz_send", optional */ mutator->afl_custom_fuzz_send = dlsym(dh, "afl_custom_fuzz_send"); if (!mutator->afl_custom_fuzz_send) { diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index eaf65987..5e352dcb 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -1954,7 +1954,8 @@ custom_mutator_stage: u32 target_len = 0; /* check if splicing makes sense yet (enough entries) */ - if (likely(afl->ready_for_splicing_count > 1)) { + if (likely(!afl->custom_splice_optout && + afl->ready_for_splicing_count > 1)) { /* Pick a random other queue entry for passing to external API that has the necessary length */ diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index b509b936..69c305f7 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -248,6 +248,8 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { PyObject_GetAttrString(py_module, "queue_get"); py_functions[PY_FUNC_FUZZ_SEND] = PyObject_GetAttrString(py_module, "fuzz_send"); + py_functions[PY_FUNC_SPLICE_OPTOUT] = + PyObject_GetAttrString(py_module, "splice_optout"); py_functions[PY_FUNC_QUEUE_NEW_ENTRY] = PyObject_GetAttrString(py_module, "queue_new_entry"); py_functions[PY_FUNC_INTROSPECTION] = @@ -394,6 +396,13 @@ void deinit_py(void *py_mutator) { } +void splice_optout_py(void *py_mutator) { + + // this is never called + (void)(py_mutator); + +} + struct custom_mutator *load_custom_mutator_py(afl_state_t *afl, char *module_name) { @@ -474,6 +483,13 @@ struct custom_mutator *load_custom_mutator_py(afl_state_t *afl, } + if (py_functions[PY_FUNC_SPLICE_OPTOUT]) { + + mutator->afl_custom_splice_optout = splice_optout_py; + afl->custom_splice_optout = 1; + + } + if (py_functions[PY_FUNC_QUEUE_NEW_ENTRY]) { mutator->afl_custom_queue_new_entry = queue_new_entry_py; -- cgit 1.4.1 From eeca3a0b2939c605497e9b3a615ee4a466f4a3f2 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 19 Jan 2023 11:52:19 +0100 Subject: lots of fixes --- custom_mutators/autotokens/TODO | 2 +- custom_mutators/autotokens/autotokens.cpp | 424 +++++++++++++++++++----------- docs/custom_mutators.md | 1 + include/afl-fuzz.h | 11 +- src/afl-fuzz-one.c | 3 +- 5 files changed, 279 insertions(+), 162 deletions(-) (limited to 'docs/custom_mutators.md') diff --git a/custom_mutators/autotokens/TODO b/custom_mutators/autotokens/TODO index 95b79373..2e39511c 100644 --- a/custom_mutators/autotokens/TODO +++ b/custom_mutators/autotokens/TODO @@ -3,4 +3,4 @@ cmplog: only add tokens that were found to fit? create from thin air if no good seed after a cycle and dict large enough? (static u32 no_of_struct_inputs;) -splicing -> check if whitespace/token is needed \ No newline at end of file +splicing -> check if whitespace/token is needed diff --git a/custom_mutators/autotokens/autotokens.cpp b/custom_mutators/autotokens/autotokens.cpp index 4f3289c9..102bea0f 100644 --- a/custom_mutators/autotokens/autotokens.cpp +++ b/custom_mutators/autotokens/autotokens.cpp @@ -38,8 +38,10 @@ typedef struct my_mutator { } my_mutator_t; -#define DEBUG \ +#undef DEBUGF +#define DEBUGF \ if (unlikely(debug)) fprintf +#define IFDEBUG if (unlikely(debug)) static afl_state *afl_ptr; static int debug = AUTOTOKENS_DEBUG; @@ -57,12 +59,12 @@ static unordered_map *> id_mapping; static unordered_map token_to_id; static unordered_map id_to_token; static string whitespace = AUTOTOKENS_WHITESPACE; +static string output; static regex *regex_comment_custom; -static regex regex_comment_star("/\\*([:print:]|\n)*?\\*/", - regex::multiline | regex::optimize); -static regex regex_word("[A-Za-z0-9_$.-]+", regex::optimize); -static regex regex_whitespace(R"([ \t]+)", regex::optimize); -static regex regex_string("\"[[:print:]]*?\"|'[[:print:]]*?'", regex::optimize); +static regex regex_comment_star("/\\*([:print:]|\n)*?\\*/", + regex::multiline | regex::optimize); +static regex regex_word("[A-Za-z0-9_$.-]+", regex::optimize); +static regex regex_whitespace(R"([ \t]+)", regex::optimize); static vector *s; // the structure of the currently selected input u32 good_whitespace_or_singleval() { @@ -104,7 +106,7 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, MAX(AUTOTOKENS_CHANGE_MIN, MIN(m_size >> 3, HAVOC_CYCLES * afl_ptr->queue_cur->perf_score * afl_ptr->havoc_div / 256)); - // DEBUG(stderr, "structure size: %lu, rounds: %u \n", m.size(), rounds); + // DEBUGF(stderr, "structure size: %lu, rounds: %u \n", m.size(), rounds); #if AUTOTOKENS_SPLICE_DISABLE == 1 #define AUTOTOKENS_MUT_MAX 12 @@ -112,7 +114,7 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, #define AUTOTOKENS_MUT_MAX 14 #endif - u32 max_rand = AUTOTOKENS_MUT_MAX; + u32 max_rand = AUTOTOKENS_MUT_MAX, new_item, pos; for (i = 0; i < rounds; ++i) { @@ -122,8 +124,8 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, case 0 ... 7: /* fall through */ { - u32 pos = rand_below(afl_ptr, m_size); - u32 cur_item = m[pos], new_item; + pos = rand_below(afl_ptr, m_size); + u32 cur_item = m[pos]; do { new_item = rand_below(afl_ptr, current_id); @@ -135,7 +137,7 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, ((whitespace_ids < new_item && whitespace_ids >= cur_item) || (whitespace_ids >= new_item && whitespace_ids < cur_item))))); - DEBUG(stderr, "MUT: %u -> %u\n", cur_item, new_item); + DEBUGF(stderr, "MUT: %u -> %u\n", cur_item, new_item); m[pos] = new_item; break; @@ -144,7 +146,6 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, /* INSERT (m_size +1 so we insert also after last place) */ case 8 ... 9: { - u32 new_item; do { new_item = rand_below(afl_ptr, current_id); @@ -154,7 +155,7 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, u32 pos = rand_below(afl_ptr, m_size + 1); m.insert(m.begin() + pos, new_item); ++m_size; - DEBUG(stderr, "INS: %u at %u\n", new_item, pos); + DEBUGF(stderr, "INS: %u at %u\n", new_item, pos); if (likely(!alternative_tokenize)) { @@ -212,7 +213,8 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, m.insert(m.begin() + dst_off, src->begin() + src_off, src->begin() + src_off + n); m_size += n; - DEBUG(stderr, "SPLICE-INS: %u at %u\n", n, dst_off); + DEBUGF(stderr, "SPLICE-INS: %u at %u\n", n, dst_off); + break; } @@ -231,13 +233,36 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, copy(src->begin() + src_off, src->begin() + src_off + n, m.begin() + dst_off); - DEBUG(stderr, "SPLICE-MUT: %u at %u\n", n, dst_off); + DEBUGF(stderr, "SPLICE-MUT: %u at %u\n", n, dst_off); break; } } + if (likely(!alternative_tokenize)) { + + // do we need a whitespace/token at the beginning? + if (dst_off && id_to_token[m[dst_off - 1]].size() > 1 && + id_to_token[m[dst_off]].size() > 1) { + + m.insert(m.begin() + dst_off, good_whitespace_or_singleval()); + ++m_size; + + } + + // do we need a whitespace/token at the end? + if (dst_off + n < m_size && + id_to_token[m[dst_off + n - 1]].size() > 1 && + id_to_token[m[dst_off + n]].size() > 1) { + + m.insert(m.begin() + dst_off + n, good_whitespace_or_singleval()); + ++m_size; + + } + + } + break; } @@ -249,11 +274,32 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, if (m_size > 8) { - m.erase(m.begin() + rand_below(afl_ptr, m_size)); - --m_size; + do { + + pos = rand_below(afl_ptr, m_size); + + } while (unlikely(pos < whitespace_ids)); + + // if what we delete will result in a missing whitespace/token, + // instead of deleting we switch the item to a whitespace or token. + if (likely(!alternative_tokenize) && pos && pos < m_size && + id_to_token[m[pos - 1]].size() > 1 && + id_to_token[m[pos + 1]].size() > 1) { + + m[pos] = good_whitespace_or_singleval(); + + } else { + + m.erase(m.begin() + pos); + --m_size; + + } } else { + // if the data is already too small do not try to make it smaller + // again this run. + max_rand = AUTOTOKENS_MUT_MAX - 2; } @@ -262,14 +308,12 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, } - // TODO: add full line insert splice, replace splace, delete - } } - string output; - u32 m_size_1 = m_size - 1; + u32 m_size_1 = m_size - 1; + output = ""; for (i = 0; i < m_size; ++i) { @@ -282,31 +326,108 @@ extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, } - u32 mutated_size = output.size(); - u8 *mutated_out = (u8 *)afl_realloc((void **)out_buf, mutated_size); + u32 mutated_size = (u32)output.size(); + u8 *mutated_out = (u8 *)output.data(); - if (unlikely(!mutated_out)) { + if (unlikely(mutated_size > max_size)) { mutated_size = max_size; } - *out_buf = NULL; - return 0; - - } - - if (unlikely(debug)) { + IFDEBUG { - DEBUG(stderr, "MUTATED to %u bytes:\n", mutated_size); + DEBUGF(stderr, "MUTATED to %u bytes:\n", mutated_size); fwrite(output.data(), 1, mutated_size, stderr); - DEBUG(stderr, "\n---\n"); + DEBUGF(stderr, "\n---\n"); } - memcpy(mutated_out, output.data(), mutated_size); *out_buf = mutated_out; ++fuzz_count; return mutated_size; } +/* I get f*cking stack overflow using C++ regex with a regex of + "\"[[:print:]]*?\"" if this matches a long string even with regex::optimize + enabled :-( */ +u8 my_search_string(string::const_iterator cur, string::const_iterator ende, + string::const_iterator *match_begin, + string::const_iterator *match_end) { + + string::const_iterator start = cur, found_begin; + u8 quote_type = 0; + + while (cur < ende) { + + switch (*cur) { + + case '"': { + + if (cur == start || *(cur - 1) != '\\') { + + if (!quote_type) { + + found_begin = cur; + quote_type = 1; + + } else if (quote_type == 1) { + + *match_begin = found_begin; + *match_end = cur + 1; + return 1; + + } + + } + + break; + + } + + case '\'': { + + if (cur == start || *(cur - 1) != '\\') { + + if (!quote_type) { + + found_begin = cur; + quote_type = 2; + + } else if (quote_type == 2) { + + *match_begin = found_begin; + *match_end = cur + 1; + return 1; + + } + + } + + break; + + } + + case '\n': + case '\r': + case 0: { + + quote_type = 0; + break; + + } + + default: + if (unlikely(quote_type && !isprint(*cur))) { quote_type = 0; } + break; + + } + + ++cur; + + } + + return 0; + +} + /* We are not using afl_custom_queue_new_entry() because not every corpus entry will be necessarily fuzzed. so we use afl_custom_queue_get() instead */ @@ -321,7 +442,7 @@ extern "C" unsigned char afl_custom_queue_get(void *data, (only_fav && !afl_ptr->queue_cur->favored)) { s = NULL; - DEBUG(stderr, "cmplog not ascii or only_fav and not favorite\n"); + DEBUGF(stderr, "cmplog not ascii or only_fav and not favorite\n"); return 0; } @@ -356,7 +477,7 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } ++extras_cnt; - DEBUG(stderr, "Added from dictionary: \"%s\"\n", ptr); + DEBUGF(stderr, "Added from dictionary: \"%s\"\n", ptr); } @@ -385,7 +506,7 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } ++a_extras_cnt; - DEBUG(stderr, "Added from auto dictionary: \"%s\"\n", ptr); + DEBUGF(stderr, "Added from auto dictionary: \"%s\"\n", ptr); } @@ -415,7 +536,7 @@ extern "C" unsigned char afl_custom_queue_get(void *data, fclose(fp); file_mapping[fn] = structure; // NULL ptr so we don't read the file again s = NULL; - DEBUG(stderr, "Too short (%lu) %s\n", len, filename); + DEBUGF(stderr, "Too short (%lu) %s\n", len, filename); return 0; } @@ -443,14 +564,14 @@ extern "C" unsigned char afl_custom_queue_get(void *data, file_mapping[fn] = NULL; s = NULL; - DEBUG(stderr, "Not text (%lu) %s\n", len, filename); + DEBUGF(stderr, "Not text (%lu) %s\n", len, filename); return 0; } } - // DEBUG(stderr, "Read %lu bytes for %s\nBefore comment trim:\n%s\n", + // DEBUGF(stderr, "Read %lu bytes for %s\nBefore comment trim:\n%s\n", // input.size(), filename, input.c_str()); if (regex_comment_custom) { @@ -463,15 +584,15 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } - DEBUG(stderr, "After replace %lu bytes for %s\n%s\n", input.size(), - filename, input.c_str()); + DEBUGF(stderr, "After replace %lu bytes for %s\n%s\n", input.size(), + filename, input.c_str()); u32 spaces = count(input.begin(), input.end(), ' '); u32 tabs = count(input.begin(), input.end(), '\t'); u32 linefeeds = count(input.begin(), input.end(), '\n'); bool ends_with_linefeed = input[input.length() - 1] == '\n'; - DEBUG(stderr, "spaces=%u tabs=%u linefeeds=%u ends=%u\n", spaces, tabs, - linefeeds, ends_with_linefeed); + DEBUGF(stderr, "spaces=%u tabs=%u linefeeds=%u ends=%u\n", spaces, tabs, + linefeeds, ends_with_linefeed); all_spaces += spaces; all_tabs += tabs; all_lf += linefeeds; @@ -479,25 +600,28 @@ extern "C" unsigned char afl_custom_queue_get(void *data, // now extract all tokens vector tokens; - smatch match; - string::const_iterator cur = input.begin(), ende = input.end(), found, prev; + string::const_iterator cur = input.begin(), ende = input.end(), found, prev, + match_begin, match_end; - DEBUG(stderr, "START!\n"); + DEBUGF(stderr, "START!\n"); if (likely(!alternative_tokenize)) { - while (regex_search(cur, ende, match, regex_string, - regex_constants::match_any | - regex_constants::match_not_null | - regex_constants::match_continuous)) { + while (my_search_string(cur, ende, &match_begin, &match_end)) { prev = cur; - found = match[0].first; - cur = match[0].second; - DEBUG(stderr, - "string %s found at start %lu offset %lu continue at %lu\n", - match[0].str().c_str(), prev - input.begin(), match.position(), - cur - input.begin()); + found = match_begin; + cur = match_end; + + IFDEBUG { + + string foo(match_begin, match_end); + DEBUGF(stderr, + "string %s found at start %lu offset %lu continue at %lu\n", + foo.c_str(), prev - input.begin(), found - prev, + cur - input.begin()); + + } if (prev < found) { // there are items between search start and find while (prev < found) { @@ -512,8 +636,8 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } tokens.push_back(std::string(start, prev)); - DEBUG(stderr, "WHITESPACE %ld \"%s\"\n", prev - start, - tokens[tokens.size() - 1].c_str()); + DEBUGF(stderr, "WHITESPACE %ld \"%s\"\n", prev - start, + tokens[tokens.size() - 1].c_str()); } else if (isalnum(*prev) || *prev == '$' || *prev == '_') { @@ -525,14 +649,14 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } - tokens.push_back(std::string(start, prev)); - DEBUG(stderr, "IDENTIFIER %ld \"%s\"\n", prev - start, - tokens[tokens.size() - 1].c_str()); + tokens.push_back(string(start, prev)); + DEBUGF(stderr, "IDENTIFIER %ld \"%s\"\n", prev - start, + tokens[tokens.size() - 1].c_str()); } else { - tokens.push_back(std::string(prev, prev + 1)); - DEBUG(stderr, "OTHER \"%c\"\n", *prev); + tokens.push_back(string(prev, prev + 1)); + DEBUGF(stderr, "OTHER \"%c\"\n", *prev); ++prev; } @@ -541,11 +665,12 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } - if (match[0].length() > 0) { tokens.push_back(match[0]); } + tokens.push_back(string(match_begin, match_end)); + DEBUGF(stderr, "TOK: %s\n", tokens[tokens.size() - 1].c_str()); } - DEBUG(stderr, "AFTER all strings\n"); + DEBUGF(stderr, "AFTER all strings\n"); if (cur < ende) { @@ -561,8 +686,8 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } tokens.push_back(std::string(start, cur)); - DEBUG(stderr, "WHITESPACE %ld \"%s\"\n", cur - start, - tokens[tokens.size() - 1].c_str()); + DEBUGF(stderr, "WHITESPACE %ld \"%s\"\n", cur - start, + tokens[tokens.size() - 1].c_str()); } else if (isalnum(*cur) || *cur == '$' || *cur == '_') { @@ -575,13 +700,13 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } tokens.push_back(std::string(start, cur)); - DEBUG(stderr, "IDENTIFIER %ld \"%s\"\n", cur - start, - tokens[tokens.size() - 1].c_str()); + DEBUGF(stderr, "IDENTIFIER %ld \"%s\"\n", cur - start, + tokens[tokens.size() - 1].c_str()); } else { tokens.push_back(std::string(cur, cur + 1)); - DEBUG(stderr, "OTHER \"%c\"\n", *cur); + DEBUGF(stderr, "OTHER \"%c\"\n", *cur); ++cur; } @@ -593,19 +718,21 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } else { // alternative tokenize - - while (regex_search(cur, ende, match, regex_string, - regex_constants::match_any | - regex_constants::match_not_null | - regex_constants::match_continuous)) { + while (my_search_string(cur, ende, &match_begin, &match_end)) { prev = cur; - found = match[0].first; - cur = match[0].second; - DEBUG(stderr, - "string %s found at start %lu offset %lu continue at %lu\n", - match[0].str().c_str(), prev - input.begin(), match.position(), - cur - input.begin()); + found = match_begin; + cur = match_end; + IFDEBUG { + + string foo(match_begin, match_end); + DEBUGF(stderr, + "string %s found at start %lu offset %lu continue at %lu\n", + foo.c_str(), prev - input.begin(), found - prev, + cur - input.begin()); + + } + if (prev < found) { // there are items between search start and find sregex_token_iterator it{prev, found, regex_whitespace, -1}; vector tokenized{it, {}}; @@ -619,10 +746,10 @@ extern "C" unsigned char afl_custom_queue_get(void *data, tokenized.end()); tokens.reserve(tokens.size() + tokenized.size() * 2 + 1); - if (unlikely(debug)) { + IFDEBUG { - DEBUG(stderr, "tokens: %lu input size: %lu\n", tokenized.size(), - input.size()); + DEBUGF(stderr, "tokens1: %lu input size: %lu\n", tokenized.size(), + input.size()); for (auto x : tokenized) { cerr << x << endl; @@ -636,10 +763,7 @@ extern "C" unsigned char afl_custom_queue_get(void *data, string::const_iterator c = token.begin(), e = token.end(), f, p; smatch m; - while (regex_search(c, e, m, regex_word, - regex_constants::match_any | - regex_constants::match_not_null | - regex_constants::match_continuous)) { + while (regex_search(c, e, m, regex_word)) { p = c; f = m[0].first; @@ -649,10 +773,10 @@ extern "C" unsigned char afl_custom_queue_get(void *data, // there are items between search start and find while (p < f) { - if (unlikely(debug)) { + IFDEBUG { string foo(p, p + 1); - DEBUG(stderr, "before string: \"%s\"\n", foo.c_str()); + DEBUGF(stderr, "before string: \"%s\"\n", foo.c_str()); } @@ -661,20 +785,21 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } - /* - string foo(p, f); - DEBUG(stderr, "before string: \"%s\"\n", - foo.c_str()); tokens.push_back(std::string(p, f)); - */ + IFDEBUG { + + string foo(p, f); + DEBUGF(stderr, "before string: \"%s\"\n", foo.c_str()); + tokens.push_back(std::string(p, f)); + + } } - DEBUG( - stderr, - "SUBstring \"%s\" found at start %lu offset %lu continue at " - "%lu\n", - m[0].str().c_str(), p - input.begin(), m.position(), - c - token.begin()); + DEBUGF(stderr, + "SUBstring \"%s\" found at start %lu offset %lu continue " + "at %lu\n", + m[0].str().c_str(), p - input.begin(), m.position(), + c - token.begin()); tokens.push_back(m[0].str()); } @@ -683,10 +808,10 @@ extern "C" unsigned char afl_custom_queue_get(void *data, while (c < e) { - if (unlikely(debug)) { + IFDEBUG { string foo(c, c + 1); - DEBUG(stderr, "after string: \"%s\"\n", foo.c_str()); + DEBUGF(stderr, "after string: \"%s\"\n", foo.c_str()); } @@ -695,17 +820,14 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } - /* - if (unlikely(debug)) { + IFDEBUG { - string foo(c, e); - DEBUG(stderr, "after string: \"%s\"\n", - foo.c_str()); + string foo(c, e); + DEBUGF(stderr, "after string: \"%s\"\n", foo.c_str()); - } + } - tokens.push_back(std::string(c, e)); - */ + tokens.push_back(std::string(c, e)); } @@ -713,7 +835,7 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } - if (match[0].length() > 0) { tokens.push_back(match[0]); } + tokens.push_back(string(match_begin, match_end)); } @@ -727,10 +849,10 @@ extern "C" unsigned char afl_custom_queue_get(void *data, tokenized.end()); tokens.reserve(tokens.size() + tokenized.size() * 2 + 1); - if (unlikely(debug)) { + IFDEBUG { - DEBUG(stderr, "tokens: %lu input size: %lu\n", tokenized.size(), - input.size()); + DEBUGF(stderr, "tokens2: %lu input size: %lu\n", tokenized.size(), + input.size()); for (auto x : tokenized) { cerr << x << endl; @@ -744,10 +866,7 @@ extern "C" unsigned char afl_custom_queue_get(void *data, string::const_iterator c = token.begin(), e = token.end(), f, p; smatch m; - while (regex_search(c, e, m, regex_word, - regex_constants::match_any | - regex_constants::match_not_null | - regex_constants::match_continuous)) { + while (regex_search(c, e, m, regex_word)) { p = c; f = m[0].first; @@ -757,10 +876,10 @@ extern "C" unsigned char afl_custom_queue_get(void *data, // there are items between search start and find while (p < f) { - if (unlikely(debug)) { + IFDEBUG { string foo(p, p + 1); - DEBUG(stderr, "before string: \"%s\"\n", foo.c_str()); + DEBUGF(stderr, "before string: \"%s\"\n", foo.c_str()); } @@ -769,25 +888,22 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } - /* - if (unlikely(debug)) { + IFDEBUG { - string foo(p, f); - DEBUG(stderr, "before string: \"%s\"\n", - foo.c_str()); + string foo(p, f); + DEBUGF(stderr, "before string: \"%s\"\n", foo.c_str()); - } + } - tokens.push_back(std::string(p, f)); - */ + tokens.push_back(std::string(p, f)); } - DEBUG(stderr, - "SUB2string \"%s\" found at start %lu offset %lu continue at " - "%lu\n", - m[0].str().c_str(), p - input.begin(), m.position(), - c - token.begin()); + DEBUGF(stderr, + "SUB2string \"%s\" found at start %lu offset %lu continue " + "at %lu\n", + m[0].str().c_str(), p - input.begin(), m.position(), + c - token.begin()); tokens.push_back(m[0].str()); } @@ -796,10 +912,10 @@ extern "C" unsigned char afl_custom_queue_get(void *data, while (c < e) { - if (unlikely(debug)) { + IFDEBUG { string foo(c, c + 1); - DEBUG(stderr, "after string: \"%s\"\n", foo.c_str()); + DEBUGF(stderr, "after string: \"%s\"\n", foo.c_str()); } @@ -808,16 +924,14 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } - /* - if (unlikely(debug)) { + IFDEBUG { - string foo(c, e); - DEBUG(stderr, "after string: \"%s\"\n", foo.c_str()); + string foo(c, e); + DEBUGF(stderr, "after string: \"%s\"\n", foo.c_str()); - } + } - tokens.push_back(std::string(c, e)); - */ + tokens.push_back(std::string(c, e)); } @@ -827,22 +941,22 @@ extern "C" unsigned char afl_custom_queue_get(void *data, } - if (unlikely(debug)) { + IFDEBUG { - DEBUG(stderr, "DUMPING TOKENS:\n"); + DEBUGF(stderr, "DUMPING TOKENS:\n"); u32 size_1 = tokens.size() - 1; for (u32 i = 0; i < tokens.size(); ++i) { - DEBUG(stderr, "%s", tokens[i].c_str()); + DEBUGF(stderr, "%s", tokens[i].c_str()); if (unlikely(alternative_tokenize && i < size_1)) { - DEBUG(stderr, "%s", whitespace.c_str()); + DEBUGF(stderr, "%s", whitespace.c_str()); } } - DEBUG(stderr, "---------------------------\n"); + DEBUGF(stderr, "---------------------------\n"); } @@ -850,7 +964,7 @@ extern "C" unsigned char afl_custom_queue_get(void *data, file_mapping[fn] = NULL; s = NULL; - DEBUG(stderr, "too few tokens\n"); + DEBUGF(stderr, "too few tokens\n"); return 0; } @@ -886,21 +1000,23 @@ extern "C" unsigned char afl_custom_queue_get(void *data, all_structure_items += structure->size(); // we are done! - DEBUG(stderr, "DONE! We have %lu tokens in the structure\n", - structure->size()); + DEBUGF(stderr, "DONE! We have %lu tokens in the structure\n", + structure->size()); + + } - } else { + else { if (entry->second == NULL) { - DEBUG(stderr, "Skipping %s\n", filename); + DEBUGF(stderr, "Skipping %s\n", filename); s = NULL; return 0; } s = entry->second; - DEBUG(stderr, "OK %s\n", filename); + DEBUGF(stderr, "OK %s\n", filename); } diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index 322caa5b..82131c92 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -150,6 +150,7 @@ def deinit(): # optional for Python sense to use it. You would only skip this if `post_process` is used to fix checksums etc. so if you are using it, e.g., as a post processing library. Note that a length > 0 *must* be returned! + The returned output buffer is under **your** memory management! - `describe` (optional): diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 1e8d085d..229bc025 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -844,15 +844,16 @@ struct custom_mutator { /** * Perform custom mutations on a given input * - * (Optional for now. Required in the future) + * (Optional) * - * @param data pointer returned in afl_custom_init by this custom mutator + * Getting an add_buf can be skipped by using afl_custom_splice_optout(). + * + * @param[in] data Pointer returned in afl_custom_init by this custom mutator * @param[in] buf Pointer to the input data to be mutated and the mutated * output * @param[in] buf_size Size of the input/output data - * @param[out] out_buf the new buffer. We may reuse *buf if large enough. - * *out_buf = NULL is treated as FATAL. - * @param[in] add_buf Buffer containing the additional test case + * @param[out] out_buf The new buffer, under your memory mgmt. + * @param[in] add_buf Buffer containing an additional test case (splicing) * @param[in] add_buf_size Size of the additional test case * @param[in] max_size Maximum size of the mutated output. The mutation must * not produce data larger than max_size. diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 5e352dcb..bd482562 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -564,8 +564,7 @@ u8 fuzz_one_original(afl_state_t *afl) { if (afl->cmplog_lvl == 3 || (afl->cmplog_lvl == 2 && afl->queue_cur->tc_ref) || afl->queue_cur->favored || - !(afl->fsrv.total_execs % afl->queued_items) || - get_cur_time() - afl->last_find_time > 300000) { // 300 seconds + get_cur_time() - afl->last_find_time > 600000) { // 600 seconds if (input_to_state_stage(afl, in_buf, out_buf, len)) { -- cgit 1.4.1 From f756734ad2782c3ed56feadb4b7b23fc82a7a968 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 13 Apr 2023 12:07:27 +0200 Subject: fix attempt at post_process implementation --- docs/Changelog.md | 1 + docs/custom_mutators.md | 9 +++++++-- include/afl-fuzz.h | 9 +++++++-- src/afl-fuzz-python.c | 49 +++++++++---------------------------------------- src/afl-fuzz-run.c | 7 ++++++- 5 files changed, 30 insertions(+), 45 deletions(-) (limited to 'docs/custom_mutators.md') diff --git a/docs/Changelog.md b/docs/Changelog.md index 501300b1..9436fc9f 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -13,6 +13,7 @@ everyone who was affected! - allow pizza mode to be disabled when AFL_PIZZA_MODE is set to -1 - option `-p mmopt` now also selects new queue items more often + - fix bug in post_process custom mutator implementation - print name of custom mutator in UI - afl-cc: - add CFI sanitizer variant to gcc targets diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index 82131c92..a1de479e 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -118,7 +118,7 @@ def deinit(): # optional for Python ### Custom Mutation -- `init`: +- `init` (optional in Python): This method is called when AFL++ starts up and is used to seed RNG and set up buffers and state. @@ -184,6 +184,11 @@ def deinit(): # optional for Python to the target, e.g. if it is too short, too corrupted, etc. If so, return a NULL buffer and zero length (or a 0 length string in Python). + NOTE: Do not make any random changes to the data in this function! + + PERFORMANCE for C/C++: If possible make the changes in-place (so modify + the `*data` directly, and return it as `*outbuf = data`. + - `fuzz_send` (optional): This method can be used if you want to send data to the target yourself, @@ -202,7 +207,7 @@ def deinit(): # optional for Python discovered if compiled with INTROSPECTION. The custom mutator can then return a string (const char *) that reports the exact mutations used. -- `deinit`: +- `deinit` (optional in Python): The last method to be called, deinitializing the state. diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 5fd393dd..8b6502b4 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -885,14 +885,19 @@ struct custom_mutator { * A post-processing function to use right before AFL writes the test case to * disk in order to execute the target. * - * (Optional) If this functionality is not needed, simply don't define this + * NOTE: Do not do any random changes to the data in this function! + * + * PERFORMANCE: If you can modify the data in-place you will have a better + * performance. Modify *data and set `*out_buf = data`. + * + * (Optional) If this functionality is not needed, simply do not define this * function. * * @param[in] data pointer returned in afl_custom_init by this custom mutator * @param[in] buf Buffer containing the test case to be executed * @param[in] buf_size Size of the test case * @param[out] out_buf Pointer to the buffer storing the test case after - * processing. External library should allocate memory for out_buf. + * processing. The external library should allocate memory for out_buf. * It can chose to alter buf in-place, if the space is large enough. * @return Size of the output buffer. */ diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 673e5a6c..7dad0770 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -219,11 +219,14 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { if (py_module != NULL) { - u8 py_notrim = 0, py_idx; - /* init, required */ + u8 py_notrim = 0; py_functions[PY_FUNC_INIT] = PyObject_GetAttrString(py_module, "init"); - if (!py_functions[PY_FUNC_INIT]) - FATAL("init function not found in python module"); + if (!py_functions[PY_FUNC_INIT]) { + + WARNF("init function not found in python module"); + + } + py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "fuzz"); if (!py_functions[PY_FUNC_FUZZ]) py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "mutate"); @@ -231,12 +234,6 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { PyObject_GetAttrString(py_module, "describe"); py_functions[PY_FUNC_FUZZ_COUNT] = PyObject_GetAttrString(py_module, "fuzz_count"); - if (!py_functions[PY_FUNC_FUZZ]) { - - WARNF("fuzz function not found in python module"); - - } - py_functions[PY_FUNC_POST_PROCESS] = PyObject_GetAttrString(py_module, "post_process"); py_functions[PY_FUNC_INIT_TRIM] = @@ -263,36 +260,6 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { if (!py_functions[PY_FUNC_DEINIT]) WARNF("deinit function not found in python module"); - for (py_idx = 0; py_idx < PY_FUNC_COUNT; ++py_idx) { - - if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { - - if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { - - // Implementing the trim API is optional for now - if (PyErr_Occurred()) { PyErr_Print(); } - py_notrim = 1; - - } else if (py_idx >= PY_OPTIONAL) { - - // Only _init and _deinit are not optional currently - - if (PyErr_Occurred()) { PyErr_Print(); } - - } else { - - fprintf(stderr, - "Cannot find/call function with index %d in external " - "Python module.\n", - py_idx); - return NULL; - - } - - } - - } - if (py_notrim) { py_functions[PY_FUNC_INIT_TRIM] = NULL; @@ -345,6 +312,8 @@ static void init_py(afl_state_t *afl, py_mutator_t *py_mutator, (void)afl; + if (py_mutator->py_functions[PY_FUNC_INIT] == NULL) { return; } + PyObject *py_args, *py_value; /* Provide the init function a seed for the Python RNG */ diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index f5425011..26e8549d 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -133,7 +133,12 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { } - if (new_mem != *mem) { *mem = new_mem; } + if (new_mem != *mem && new_mem != NULL && new_size > 0) { + + *mem = afl_realloc((void **)mem, new_size); + memmove(*mem, new_mem, new_size); + + } if (unlikely(afl->custom_mutators_count)) { -- cgit 1.4.1