From 7b59e05600aff4e9deb3110887d6ad49838f02b0 Mon Sep 17 00:00:00 2001 From: h1994st Date: Mon, 2 Mar 2020 15:27:29 -0500 Subject: Add new APIs for the custom mutator --- src/afl-fuzz.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) (limited to 'src/afl-fuzz.c') diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index c7f8ccad..035f74dc 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -155,7 +155,7 @@ static void usage(u8* argv0, int more_help) { "LD_BIND_LAZY: do not set LD_BIND_NOW env var for target\n" "AFL_BENCH_JUST_ONE: run the target just once\n" "AFL_DUMB_FORKSRV: use fork server without feedback from target\n" - "AFL_CUSTOM_MUTATOR_LIBRARY: lib with afl_custom_mutator() to mutate inputs\n" + "AFL_CUSTOM_MUTATOR_LIBRARY: lib with afl_custom_fuzz() to mutate inputs\n" "AFL_CUSTOM_MUTATOR_ONLY: avoid AFL++'s internal mutators\n" "AFL_PYTHON_MODULE: mutate and trim inputs with the specified Python module\n" "AFL_PYTHON_ONLY: skip AFL++'s own mutators\n" @@ -864,13 +864,6 @@ int main(int argc, char** argv, char** envp) { setup_dirs_fds(); -#ifdef USE_PYTHON - if (init_py()) FATAL("Failed to initialize Python module"); -#else - if (getenv("AFL_PYTHON_MODULE")) - FATAL("Your AFL binary was built without Python support"); -#endif - setup_cmdline_file(argv + optind); read_testcases(); @@ -1147,13 +1140,10 @@ stop_fuzzing: destroy_extras(); ck_free(target_path); ck_free(sync_id); + destroy_custom_mutator(); alloc_report(); -#ifdef USE_PYTHON - finalize_py(); -#endif - OKF("We're done here. Have a nice day!\n"); exit(0); -- cgit 1.4.1 From 90506479e7de57c97d97958c61b2513009687d90 Mon Sep 17 00:00:00 2001 From: h1994st Date: Mon, 2 Mar 2020 21:30:10 -0500 Subject: Refactoring `fuzz_py` API --- include/afl-fuzz.h | 4 ++- src/afl-fuzz-mutators.c | 1 - src/afl-fuzz-one.c | 2 +- src/afl-fuzz-python.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++-- src/afl-fuzz.c | 7 +++-- 5 files changed, 85 insertions(+), 9 deletions(-) (limited to 'src/afl-fuzz.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index e6aaee68..27b22082 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -627,7 +627,9 @@ int init_py_module(u8*); void finalize_py_module(); void init_py(unsigned int seed); -void fuzz_py(char*, size_t, char*, size_t, char**, size_t*); +/* TODO: unify fuzz interface for custom mutator and Python mutator */ +size_t fuzz_py(u8*, size_t, u8*, size_t, unsigned int); +void fuzz_py_original(char*, size_t, char*, size_t, char**, size_t*); size_t pre_save_py(u8* data, size_t size, u8** new_data); u32 init_trim_py(u8*, size_t); u32 post_trim_py(u8); diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 22ca1384..9365d487 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -281,7 +281,6 @@ void load_custom_mutator_py(const char* module_name) { mutator->name = module_name; ACTF("Loading Python mutator library from '%s'...", module_name); - /* TODO: unify "init" and "fuzz" */ if (py_functions[PY_FUNC_INIT]) mutator->afl_custom_init = init_py; diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 24ea76f3..1e6dd45d 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -1647,7 +1647,7 @@ python_stage: ck_read(fd, new_buf, target->len, target->fname); close(fd); - fuzz_py(out_buf, len, new_buf, target->len, &retbuf, &retlen); + fuzz_py_original(out_buf, len, new_buf, target->len, &retbuf, &retlen); ck_free(new_buf); diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 30156fa6..c8caa4c1 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -32,6 +32,8 @@ int init_py_module(u8* module_name) { if (!module_name) return 1; + Py_Initialize(); + #if PY_MAJOR_VERSION >= 3 PyObject* py_name = PyUnicode_FromString(module_name); #else @@ -58,7 +60,12 @@ int init_py_module(u8* module_name) { if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { - if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { + if (py_idx == PY_FUNC_PRE_SAVE) { + + // Implenting the pre_save API is optional for now + if (PyErr_Occurred()) PyErr_Print(); + + } else if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { // Implementing the trim API is optional for now if (PyErr_Occurred()) PyErr_Print(); @@ -152,8 +159,9 @@ void init_py(unsigned int seed) { } } -void fuzz_py(char* buf, size_t buflen, char* add_buf, size_t add_buflen, - char** ret, size_t* retlen) { +void fuzz_py_original(char* buf, size_t buflen, + char* add_buf, size_t add_buflen, + char** ret, size_t* retlen) { if (py_module != NULL) { @@ -204,6 +212,72 @@ void fuzz_py(char* buf, size_t buflen, char* add_buf, size_t add_buflen, } +size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, + unsigned int seed) { + + size_t out_size; + PyObject *py_args, *py_value; + py_args = PyTuple_New(3); + + py_value = PyByteArray_FromStringAndSize(data, size); + if (!py_value) { + + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + + } + + PyTuple_SetItem(py_args, 0, py_value); + +#if PY_MAJOR_VERSION >= 3 + py_value = PyLong_FromLong(max_size); +#else + py_value = PyInt_FromLong(max_size); +#endif + if (!py_value) { + + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + + } + + PyTuple_SetItem(py_args, 1, py_value); + +#if PY_MAJOR_VERSION >= 3 + py_value = PyLong_FromLong(seed); +#else + py_value = PyInt_FromLong(seed); +#endif + if (!py_value) { + + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + + } + + PyTuple_SetItem(py_args, 2, py_value); + + py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args); + + Py_DECREF(py_args); + + if (py_value != NULL) { + + out_size = PyByteArray_Size(py_value); + memcpy(mutated_out, PyByteArray_AsString(py_value), out_size); + Py_DECREF(py_value); + + return out_size; + + } else { + + PyErr_Print(); + FATAL("Call failed"); + + } + +} + size_t pre_save_py(u8* data, size_t size, u8** new_data) { size_t new_size; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 2f0043ab..d329a20e 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -752,9 +752,9 @@ int main(int argc, char** argv, char** envp) { if (get_afl_env("AFL_FAST_CAL")) fast_cal = 1; if (get_afl_env("AFL_AUTORESUME")) { - + autoresume = 1; - if (in_place_resume) + if (in_place_resume) SAYF("AFL_AUTORESUME has no effect for '-i -'"); } @@ -862,7 +862,6 @@ int main(int argc, char** argv, char** envp) { check_cpu_governor(); setup_post(); - setup_custom_mutator(); setup_shm(dumb_mode); if (!in_bitmap) memset(virgin_bits, 255, MAP_SIZE); @@ -873,6 +872,8 @@ int main(int argc, char** argv, char** envp) { setup_dirs_fds(); + setup_custom_mutator(); + setup_cmdline_file(argv + optind); read_testcases(); -- cgit 1.4.1 From df465216583afcc0e65e4468e6383afd7a688ddc Mon Sep 17 00:00:00 2001 From: h1994st Date: Tue, 3 Mar 2020 19:48:13 -0500 Subject: Finish refactoring APIs for the custom mutator and Python module - Remove AFL_PYTHON_ONLY (env) and python_only (variable) - Unify fuzz API of the custom mutator and Python module - Merge the custom mutator into the old python_stage, which is now renamed to custom_mutator_stage --- .gitignore | 2 + docs/custom_mutator.md | 2 +- docs/env_variables.md | 34 +++++++------- include/afl-fuzz.h | 50 +++++++++++--------- include/envs.h | 2 +- src/afl-fuzz-globals.c | 3 +- src/afl-fuzz-mutators.c | 4 +- src/afl-fuzz-one.c | 121 ++++++++++++------------------------------------ src/afl-fuzz-python.c | 108 +++++++++++------------------------------- src/afl-fuzz-run.c | 3 +- src/afl-fuzz.c | 18 ++----- 11 files changed, 115 insertions(+), 232 deletions(-) (limited to 'src/afl-fuzz.c') diff --git a/.gitignore b/.gitignore index 4032b826..bdd342c4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,9 +18,11 @@ afl-qemu-trace afl-showmap afl-tmin afl-analyze.8 +afl-as.8 afl-clang-fast++.8 afl-clang-fast.8 afl-cmin.8 +afl-cmin.bash.8 afl-fuzz.8 afl-gcc.8 afl-gcc-fast.8 diff --git a/docs/custom_mutator.md b/docs/custom_mutator.md index dff32c1d..68e27de7 100644 --- a/docs/custom_mutator.md +++ b/docs/custom_mutator.md @@ -13,7 +13,7 @@ a given grammar. The custom mutator library is passed to afl-fuzz via the AFL_CUSTOM_MUTATOR_LIBRARY environment variable. The library must export -the afl_custom_mutator() function and must be compiled as a shared object. +the afl_custom_fuzz() function and must be compiled as a shared object. For example: ``` $CC -shared -Wall -O3 .c -o .so diff --git a/docs/env_variables.md b/docs/env_variables.md index 527f1c1b..5214f808 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -104,7 +104,7 @@ Then there are a few specific features that are only available in llvm_mode: - Setting AFL_LLVM_LAF_SPLIT_COMPARES will split all floating point and 64, 32 and 16 bit integer CMP instructions - See llvm_mode/README.laf-intel.md for more information. + See llvm_mode/README.laf-intel.md for more information. ### WHITELIST @@ -192,7 +192,7 @@ checks or alter some of the more exotic semantics of the tool: deciding if a particular test case is a "hang". The default is 1 second or the value of the -t parameter, whichever is larger. Dialing the value down can be useful if you are very concerned about slow inputs, or if you - don't want AFL to spend too much time classifying that stuff and just + don't want AFL to spend too much time classifying that stuff and just rapidly put all timeouts in that bin. - AFL_NO_ARITH causes AFL to skip most of the deterministic arithmetics. @@ -223,15 +223,15 @@ checks or alter some of the more exotic semantics of the tool: for more. - Setting AFL_CUSTOM_MUTATOR_LIBRARY to a shared library with - afl_custom_mutator() creates additional mutations through this library. + afl_custom_fuzz() creates additional mutations through this library. + If afl-fuzz is compiled with Python (which is autodetected during builing + afl-fuzz), setting AFL_PYTHON_MODULE to a Python module can also provide + additional mutations. If AFL_CUSTOM_MUTATOR_ONLY is also set, all mutations will solely be - performed with/from the library. See [custom_mutator.md](custom_mutator.md) - - - For AFL_PYTHON_MODULE and AFL_PYTHON_ONLY - they require afl-fuzz to - be compiled with Python (which is autodetected during builing afl-fuzz). - Please see [python_mutators.md](python_mutators.md). + performed with/from the library/Python module. This feature allows to configure custom mutators which can be very helpful in e.g. fuzzing XML or other highly flexible structured input. + Please see [custom_mutator.md](custom_mutator.md) or [python_mutators.md](python_mutators.md). - AFL_FAST_CAL keeps the calibration stage about 2.5x faster (albeit less precise), which can help when starting a session against a slow target. @@ -283,7 +283,7 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - Setting AFL_INST_LIBS causes the translator to also instrument the code inside any dynamically linked libraries (notably including glibc). - + - Setting AFL_COMPCOV_LEVEL enables the CompareCoverage tracing of all cmp and sub in x86 and x86_64 and memory comparions functions (e.g. strcmp, memcmp, ...) when libcompcov is preloaded using AFL_PRELOAD. @@ -292,7 +292,7 @@ The QEMU wrapper used to instrument binary-only code supports several settings: only comparisons with immediate values / read-only memory and AFL_COMPCOV_LEVEL=2 that instruments all the comparions. Level 2 is more accurate but may need a larger shared memory. - + - Setting AFL_QEMU_COMPCOV enables the CompareCoverage tracing of all cmp and sub in x86 and x86_64. This is an alias of AFL_COMPCOV_LEVEL=1 when AFL_COMPCOV_LEVEL is @@ -304,25 +304,25 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - AFL_DEBUG will print the found entrypoint for the binary to stderr. Use this if you are unsure if the entrypoint might be wrong - but - use it directly, e.g. afl-qemu-trace ./program + use it directly, e.g. afl-qemu-trace ./program - AFL_ENTRYPOINT allows you to specify a specific entrypoint into the binary (this can be very good for the performance!). The entrypoint is specified as hex address, e.g. 0x4004110 Note that the address must be the address of a basic block. - + - When the target is i386/x86_64 you can specify the address of the function that has to be the body of the persistent loop using AFL_QEMU_PERSISTENT_ADDR=`start addr`. - + - Another modality to execute the persistent loop is to specify also the AFL_QEMU_PERSISTENT_RET=`end addr` env variable. With this variable assigned, instead of patching the return address, the specified instruction is transformed to a jump towards `start addr`. - + - AFL_QEMU_PERSISTENT_GPR=1 QEMU will save the original value of general purpose registers and restore them in each persistent cycle. - + - With AFL_QEMU_PERSISTENT_RETADDR_OFFSET you can specify the offset from the stack pointer in which QEMU can find the return address when `start addr` is hitted. @@ -376,7 +376,7 @@ The library honors these environmental variables: - AFL_LD_NO_CALLOC_OVER inhibits abort() on calloc() overflows. Most of the common allocators check for that internally and return NULL, so it's a security risk only in more exotic setups. - + - AFL_ALIGNED_ALLOC=1 will force the alignment of the allocation size to max_align_t to be compliant with the C standard. @@ -410,7 +410,7 @@ optimal values if not already present in the environment: - In the same vein, by default, MSAN_OPTIONS are set to: - exit_code=86 (required for legacy reasons) + exit_code=86 (required for legacy reasons) abort_on_error=1 symbolize=0 msan_track_origins=0 diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 27b22082..37b6832c 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -276,8 +276,7 @@ extern u8 cal_cycles, /* Calibration cycles defaults */ no_unlink, /* do not unlink cur_input */ use_stdin, /* use stdin for sending data */ debug, /* Debug mode */ - custom_only, /* Custom mutator only mode */ - python_only; /* Python-only mode */ + custom_only; /* Custom mutator only mode */ extern u32 stats_update_freq; /* Stats update frequency (execs) */ @@ -471,6 +470,8 @@ struct custom_mutator { * Initialize the custom mutator. * * (Optional) + * + * @param seed Seed used for the mutation. */ void (*afl_custom_init)(unsigned int seed); @@ -479,17 +480,18 @@ struct custom_mutator { * * (Optional for now. Required in the future) * - * @param[in] data Input data to be mutated - * @param[in] size Size of input data + * @param[in] buf Input data to be mutated + * @param[in] buf_size Size of input data + * @param[in] add_buf Buffer containing the additional test case + * @param[in] add_buf_size Size of the additional test case * @param[out] mutated_out Buffer to store the mutated input * @param[in] max_size Maximum size of the mutated output. The mutation must not * produce data larger than max_size. - * @param[in] seed Seed used for the mutation. The mutation should produce the - * same output given the same seed. * @return Size of the mutated output. */ - size_t (*afl_custom_fuzz)(u8* data, size_t size, u8* mutated_out, - size_t max_size, unsigned int seed); + size_t (*afl_custom_fuzz)(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size); /** * A post-processing function to use right before AFL writes the test case to @@ -498,12 +500,14 @@ struct custom_mutator { * (Optional) If this functionality is not needed, simply don't define this * function. * - * @param[in] data Buffer containing the test case to be executed - * @param[in] size Size of the test case - * @param[out] new_data Buffer to store the test case after processing - * @return Size of data after processing + * @param[in] buf Buffer containing the test case to be executed + * @param[in] buf_size Size of the test case + * @param[out] out_buf Pointer to the buffer of storing the test case after + * processing. External library should allocate memory for out_buf. AFL++ + * will release the memory after saving the test case. + * @return Size of the output buffer after processing */ - size_t (*afl_custom_pre_save)(u8* data, size_t size, u8** new_data); + size_t (*afl_custom_pre_save)(u8* buf, size_t buf_size, u8** out_buf); /** * This method is called at the start of each trimming operation and receives @@ -521,11 +525,11 @@ struct custom_mutator { * * (Optional) * - * @param data Buffer containing the test case - * @param size Size of the test case + * @param buf Buffer containing the test case + * @param buf_size Size of the test case * @return The amount of possible iteration steps to trim the input */ - u32 (*afl_custom_init_trim)(u8* data, size_t size); + u32 (*afl_custom_init_trim)(u8* buf, size_t buf_size); /** * This method is called for each trimming operation. It doesn't have any @@ -538,10 +542,12 @@ struct custom_mutator { * * (Optional) * - * @param[out] ret Buffer containing the trimmed test case - * @param[out] ret_len Size of the trimmed test case + * @param[out] out_buf Pointer to the buffer containing the trimmed test case. + * External library should allocate memory for out_buf. AFL++ will release + * the memory after saving the test case. + * @param[out] out_buf_size Pointer to the size of the trimmed test case */ - void (*afl_custom_trim)(u8** ret, size_t* ret_len); + void (*afl_custom_trim)(u8** out_buf, size_t* out_buf_size); /** * This method is called after each trim operation to inform you if your @@ -627,9 +633,9 @@ int init_py_module(u8*); void finalize_py_module(); void init_py(unsigned int seed); -/* TODO: unify fuzz interface for custom mutator and Python mutator */ -size_t fuzz_py(u8*, size_t, u8*, size_t, unsigned int); -void fuzz_py_original(char*, size_t, char*, size_t, char**, size_t*); +size_t fuzz_py(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size); size_t pre_save_py(u8* data, size_t size, u8** new_data); u32 init_trim_py(u8*, size_t); u32 post_trim_py(u8); diff --git a/include/envs.h b/include/envs.h index 306143be..791887d7 100644 --- a/include/envs.h +++ b/include/envs.h @@ -24,7 +24,7 @@ const char *afl_environment_variables[] = { "AFL_NO_X86", // not really an env but we dont want to warn on it "AFL_PATH", "AFL_PERFORMANCE_FILE", //"AFL_PERSISTENT", // not implemented anymore, so warn additionally - "AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE", "AFL_PYTHON_ONLY", + "AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE", "AFL_QEMU_COMPCOV", "AFL_QEMU_COMPCOV_DEBUG", "AFL_QEMU_DEBUG_MAPS", "AFL_QEMU_DISABLE_CACHE", "AFL_QEMU_PERSISTENT_ADDR", "AFL_QEMU_PERSISTENT_CNT", "AFL_QEMU_PERSISTENT_GPR", diff --git a/src/afl-fuzz-globals.c b/src/afl-fuzz-globals.c index a054499f..87418753 100644 --- a/src/afl-fuzz-globals.c +++ b/src/afl-fuzz-globals.c @@ -88,8 +88,7 @@ u8 cal_cycles = CAL_CYCLES, /* Calibration cycles defaults */ no_unlink, /* do not unlink cur_input */ use_stdin = 1, /* use stdin for sending data */ be_quiet, /* is AFL_QUIET set? */ - custom_only, /* Custom mutator only mode */ - python_only; /* Python-only mode */ + custom_only; /* Custom mutator only mode */ u32 stats_update_freq = 1; /* Stats update frequency (execs) */ diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 9365d487..26eaea59 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -27,6 +27,7 @@ void setup_custom_mutator(void) { + /* Try mutator library first */ u8* fn = getenv("AFL_CUSTOM_MUTATOR_LIBRARY"); if (fn) { @@ -41,6 +42,7 @@ void setup_custom_mutator(void) { return; } + /* Try Python module */ #ifdef USE_PYTHON u8* module_name = getenv("AFL_PYTHON_MODULE"); @@ -286,7 +288,7 @@ void load_custom_mutator_py(const char* module_name) { /* "afl_custom_fuzz" should not be NULL, but the interface of Python mutator is quite different from the custom mutator. */ - mutator->afl_custom_fuzz = NULL; + mutator->afl_custom_fuzz = fuzz_py; if (py_functions[PY_FUNC_PRE_SAVE]) mutator->afl_custom_pre_save = pre_save_py; diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 1e6dd45d..5d00e8df 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -482,56 +482,6 @@ u8 fuzz_one_original(char** argv) { if (use_radamsa > 1) goto radamsa_stage; - // custom_stage: // not used - yet - - if (mutator->afl_custom_fuzz) { - - stage_short = "custom"; - stage_name = "custom mutator"; - stage_max = len << 3; - stage_val_type = STAGE_VAL_NONE; - - const u32 max_seed_size = 4096 * 4096; - u8* mutated_buf = ck_alloc(max_seed_size); - - orig_hit_cnt = queued_paths + unique_crashes; - - for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) { - - size_t orig_size = (size_t)len; - size_t mutated_size = mutator->afl_custom_fuzz(in_buf, orig_size, - mutated_buf, max_seed_size, - UR(UINT32_MAX)); - if (mutated_size > 0) { - - out_buf = ck_realloc(out_buf, mutated_size); - memcpy(out_buf, mutated_buf, mutated_size); - if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) { - - goto abandon_entry; - - } - - } - - } - - ck_free(mutated_buf); - new_hit_cnt = queued_paths + unique_crashes; - - stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt; - stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max; - - if (custom_only) { - - /* Skip other stages */ - ret_val = 0; - goto abandon_entry; - - } - - } - if (cmplog_mode) { if (input_to_state_stage(argv, in_buf, out_buf, len, queue_cur->exec_cksum)) @@ -551,11 +501,7 @@ u8 fuzz_one_original(char** argv) { : havoc_max_mult * 100)) || queue_cur->passed_det) { -#ifdef USE_PYTHON - goto python_stage; -#else - goto havoc_stage; -#endif + goto custom_mutator_stage; } @@ -564,11 +510,7 @@ u8 fuzz_one_original(char** argv) { if (master_max && (queue_cur->exec_cksum % master_max) != master_id - 1) { -#ifdef USE_PYTHON - goto python_stage; -#else - goto havoc_stage; -#endif + goto custom_mutator_stage; } @@ -1583,24 +1525,25 @@ skip_extras: if (!queue_cur->passed_det) mark_as_det_done(queue_cur); -#ifdef USE_PYTHON -python_stage: - /********************************** - * EXTERNAL MUTATORS (Python API) * - **********************************/ +custom_mutator_stage: + /******************* + * CUSTOM MUTATORS * + *******************/ - if (!py_module) goto havoc_stage; + if (!mutator) goto havoc_stage; + if (!mutator->afl_custom_fuzz) goto havoc_stage; - stage_name = "python"; - stage_short = "python"; + stage_name = "custom mutator"; + stage_short = "custom"; stage_max = HAVOC_CYCLES * perf_score / havoc_div / 100; + stage_val_type = STAGE_VAL_NONE; if (stage_max < HAVOC_MIN) stage_max = HAVOC_MIN; - orig_hit_cnt = queued_paths + unique_crashes; + const u32 max_seed_size = 4096 * 4096; + u8* mutated_buf = ck_alloc(max_seed_size); - char* retbuf = NULL; - size_t retlen = 0; + orig_hit_cnt = queued_paths + unique_crashes; for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) { @@ -1647,26 +1590,24 @@ python_stage: ck_read(fd, new_buf, target->len, target->fname); close(fd); - fuzz_py_original(out_buf, len, new_buf, target->len, &retbuf, &retlen); + size_t mutated_size = mutator->afl_custom_fuzz(out_buf, len, + new_buf, target->len, + mutated_buf, max_seed_size); ck_free(new_buf); - if (retbuf) { + if (mutated_size > 0) { - if (!retlen) goto abandon_entry; + out_buf = ck_realloc(out_buf, mutated_size); + memcpy(out_buf, mutated_buf, mutated_size); - if (common_fuzz_stuff(argv, retbuf, retlen)) { + if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) { - free(retbuf); + ck_free(mutated_buf); goto abandon_entry; } - /* Reset retbuf/retlen */ - free(retbuf); - retbuf = NULL; - retlen = 0; - /* If we're finding new stuff, let's run for a bit longer, limits permitting. */ @@ -1687,12 +1628,13 @@ python_stage: } + ck_free(mutated_buf); new_hit_cnt = queued_paths + unique_crashes; - stage_finds[STAGE_PYTHON] += new_hit_cnt - orig_hit_cnt; - stage_cycles[STAGE_PYTHON] += stage_max; + stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt; + stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max; - if (python_only) { + if (custom_only) { /* Skip other stages */ ret_val = 0; @@ -1700,8 +1642,6 @@ python_stage: } -#endif - /**************** * RANDOM HAVOC * ****************/ @@ -2270,11 +2210,10 @@ retry_splicing: out_buf = ck_alloc_nozero(len); memcpy(out_buf, in_buf, len); -#ifdef USE_PYTHON - goto python_stage; -#else - goto havoc_stage; -#endif + goto custom_mutator_stage; + /* ???: While integrating Python module, the author decided to jump to + python stage, but the reason behind this is not clear.*/ + // goto havoc_stage; } diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index c8caa4c1..c22e4402 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -159,67 +159,16 @@ void init_py(unsigned int seed) { } } -void fuzz_py_original(char* buf, size_t buflen, - char* add_buf, size_t add_buflen, - char** ret, size_t* retlen) { +size_t fuzz_py(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size) { - if (py_module != NULL) { - - PyObject *py_args, *py_value; - py_args = PyTuple_New(2); - py_value = PyByteArray_FromStringAndSize(buf, buflen); - if (!py_value) { - - Py_DECREF(py_args); - fprintf(stderr, "Cannot convert argument\n"); - return; - - } - - PyTuple_SetItem(py_args, 0, py_value); - - py_value = PyByteArray_FromStringAndSize(add_buf, add_buflen); - if (!py_value) { - - Py_DECREF(py_args); - fprintf(stderr, "Cannot convert argument\n"); - return; - - } - - PyTuple_SetItem(py_args, 1, py_value); - - py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args); - - Py_DECREF(py_args); - - if (py_value != NULL) { - - *retlen = PyByteArray_Size(py_value); - *ret = malloc(*retlen); - memcpy(*ret, PyByteArray_AsString(py_value), *retlen); - Py_DECREF(py_value); - - } else { - - PyErr_Print(); - fprintf(stderr, "Call failed\n"); - return; - - } - - } - -} - -size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, - unsigned int seed) { - - size_t out_size; + size_t mutated_size; PyObject *py_args, *py_value; py_args = PyTuple_New(3); - py_value = PyByteArray_FromStringAndSize(data, size); + /* buf */ + py_value = PyByteArray_FromStringAndSize(buf, buf_size); if (!py_value) { Py_DECREF(py_args); @@ -229,11 +178,8 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, PyTuple_SetItem(py_args, 0, py_value); -#if PY_MAJOR_VERSION >= 3 - py_value = PyLong_FromLong(max_size); -#else - py_value = PyInt_FromLong(max_size); -#endif + /* add_buf */ + py_value = PyByteArray_FromStringAndSize(add_buf, add_buf_size); if (!py_value) { Py_DECREF(py_args); @@ -243,10 +189,11 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, PyTuple_SetItem(py_args, 1, py_value); + /* max_size */ #if PY_MAJOR_VERSION >= 3 - py_value = PyLong_FromLong(seed); + py_value = PyLong_FromLong(max_size); #else - py_value = PyInt_FromLong(seed); + py_value = PyInt_FromLong(max_size); #endif if (!py_value) { @@ -263,11 +210,10 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, if (py_value != NULL) { - out_size = PyByteArray_Size(py_value); - memcpy(mutated_out, PyByteArray_AsString(py_value), out_size); + mutated_size = PyByteArray_Size(py_value); + memcpy(mutated_out, PyByteArray_AsString(py_value), mutated_size); Py_DECREF(py_value); - - return out_size; + return mutated_size; } else { @@ -278,12 +224,12 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, } -size_t pre_save_py(u8* data, size_t size, u8** new_data) { +size_t pre_save_py(u8* buf, size_t buf_size, u8** out_buf) { - size_t new_size; + size_t out_buf_size; PyObject *py_args, *py_value; py_args = PyTuple_New(2); - py_value = PyByteArray_FromStringAndSize(data, size); + py_value = PyByteArray_FromStringAndSize(buf, buf_size); if (!py_value) { Py_DECREF(py_args); @@ -299,11 +245,11 @@ size_t pre_save_py(u8* data, size_t size, u8** new_data) { if (py_value != NULL) { - new_size = PyByteArray_Size(py_value); - *new_data = malloc(new_size); - memcpy(*new_data, PyByteArray_AsString(py_value), new_size); + out_buf_size = PyByteArray_Size(py_value); + *out_buf = malloc(out_buf_size); + memcpy(*out_buf, PyByteArray_AsString(py_value), out_buf_size); Py_DECREF(py_value); - return new_size; + return out_buf_size; } else { @@ -314,12 +260,12 @@ size_t pre_save_py(u8* data, size_t size, u8** new_data) { } -u32 init_trim_py(u8* buf, size_t buflen) { +u32 init_trim_py(u8* buf, size_t buf_size) { PyObject *py_args, *py_value; py_args = PyTuple_New(1); - py_value = PyByteArray_FromStringAndSize(buf, buflen); + py_value = PyByteArray_FromStringAndSize(buf, buf_size); if (!py_value) { Py_DECREF(py_args); @@ -389,7 +335,7 @@ u32 post_trim_py(u8 success) { } -void trim_py(u8** ret, size_t* retlen) { +void trim_py(u8** out_buf, size_t* out_buf_size) { PyObject *py_args, *py_value; @@ -399,9 +345,9 @@ void trim_py(u8** ret, size_t* retlen) { if (py_value != NULL) { - *retlen = PyByteArray_Size(py_value); - *ret = malloc(*retlen); - memcpy(*ret, PyByteArray_AsString(py_value), *retlen); + *out_buf_size = PyByteArray_Size(py_value); + *out_buf = malloc(*out_buf_size); + memcpy(*out_buf, PyByteArray_AsString(py_value), *out_buf_size); Py_DECREF(py_value); } else { diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index a1d10387..12352355 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -309,11 +309,12 @@ void write_to_testcase(void* mem, u32 len) { lseek(fd, 0, SEEK_SET); - if (mutator->afl_custom_pre_save) { + if (mutator && mutator->afl_custom_pre_save) { u8* new_data; size_t new_size = mutator->afl_custom_pre_save(mem, len, &new_data); ck_write(fd, new_data, new_size, out_file); + ck_free(new_data); } else { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index d329a20e..a96ee1d0 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -158,7 +158,6 @@ static void usage(u8* argv0, int more_help) { "AFL_CUSTOM_MUTATOR_LIBRARY: lib with afl_custom_fuzz() to mutate inputs\n" "AFL_CUSTOM_MUTATOR_ONLY: avoid AFL++'s internal mutators\n" "AFL_PYTHON_MODULE: mutate and trim inputs with the specified Python module\n" - "AFL_PYTHON_ONLY: skip AFL++'s own mutators\n" "AFL_DEBUG: extra debugging output for Python mode trimming\n" "AFL_DISABLE_TRIM: disable the trimming of test cases\n" "AFL_NO_UI: switch status screen off\n" @@ -658,11 +657,10 @@ int main(int argc, char** argv, char** envp) { OKF("afl-tmin fork server patch from github.com/nccgroup/TriforceAFL"); OKF("MOpt Mutator from github.com/puppet-meteor/MOpt-AFL"); - if (sync_id && force_deterministic && - (getenv("AFL_CUSTOM_MUTATOR_ONLY") || getenv("AFL_PYTHON_ONLY"))) + if (sync_id && force_deterministic && getenv("AFL_CUSTOM_MUTATOR_ONLY")) WARNF( - "Using -M master with the AFL_..._ONLY mutator options will result in " - "no deterministic mutations being done!"); + "Using -M master with the AFL_CUSTOM_MUTATOR_ONLY mutator options will " + "result in no deterministic mutations being done!"); check_environment_vars(envp); @@ -832,16 +830,6 @@ int main(int argc, char** argv, char** envp) { if (get_afl_env("AFL_DEBUG")) debug = 1; - if (get_afl_env("AFL_PYTHON_ONLY")) { - - /* This ensures we don't proceed to havoc/splice */ - python_only = 1; - - /* Ensure we also skip all deterministic steps */ - skip_deterministic = 1; - - } - if (get_afl_env("AFL_CUSTOM_MUTATOR_ONLY")) { /* This ensures we don't proceed to havoc/splice */ -- cgit 1.4.1 From 445d4b7e594ba6933d69ef680ea7b3a64c214d82 Mon Sep 17 00:00:00 2001 From: h1994st Date: Tue, 3 Mar 2020 23:17:24 -0500 Subject: Update the documents of the custom mutator - Merge python_mutators.md into custom_mutator.md - Remove python_mutators.md --- docs/custom_mutator.md | 218 +++++++++++++++++++++++++++++++++++++++++------- docs/env_variables.md | 8 +- docs/python_mutators.md | 148 -------------------------------- src/afl-fuzz.c | 2 +- 4 files changed, 192 insertions(+), 184 deletions(-) delete mode 100644 docs/python_mutators.md (limited to 'src/afl-fuzz.c') diff --git a/docs/custom_mutator.md b/docs/custom_mutator.md index 68e27de7..2a6c365d 100644 --- a/docs/custom_mutator.md +++ b/docs/custom_mutator.md @@ -1,45 +1,201 @@ -# Adding custom mutators to AFL +# Custom Mutators in AFL++ This file describes how you can implement custom mutations to be used in AFL. +For now, we support C/C++ library and Python module, collectivelly named as the +custom mutator. -Implemented by Khaled Yakdan from Code Intelligence +Implemented by +- C/C++ library (`*.so`): Khaled Yakdan from Code Intelligence () +- Python module: Christian Holler from Mozilla () -## 1) Description +## 1) Introduction -Custom mutator libraries can be passed to afl-fuzz to perform custom mutations -on test cases beyond those available in AFL - for example, to enable -structure-aware fuzzing by using libraries that perform mutations according to -a given grammar. +Custom mutators can be passed to `afl-fuzz` to perform custom mutations on test +cases beyond those available in AFL. For example, to enable structure-aware +fuzzing by using libraries that perform mutations according to a given grammar. -The custom mutator library is passed to afl-fuzz via the -AFL_CUSTOM_MUTATOR_LIBRARY environment variable. The library must export -the afl_custom_fuzz() function and must be compiled as a shared object. -For example: +The custom mutator is passed to `afl-fuzz` via the `AFL_CUSTOM_MUTATOR_LIBRARY` +or `AFL_PYTHON_MODULE` environment variable., and must export a fuzz function. +Please see [APIs](#2-apis) and [Usage](#3-usage) for detail. + +The custom mutation stage is set to be the first non-deterministic stage (right before the havoc stage). + +Note: If `AFL_CUSTOM_MUTATOR_ONLY` is set, all mutations will solely be +performed with the custom mutator. + +## 2) APIs + +C/C++: +```c +void afl_custom_init(unsigned int seed); +size_t afl_custom_fuzz(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size); +size_t afl_custom_pre_save(u8* buf, size_t buf_size, u8** out_buf); +u32 afl_custom_init_trim(u8* buf, size_t buf_size); +void afl_custom_trim(u8** out_buf, size_t* out_buf_size); +u32 afl_custom_post_trim(u8 success); +``` + +Python: +```python +def init(seed): + pass + +def fuzz(buf, add_buf, max_size): + return mutated_out + +def pre_save(buf): + return out_buf + +def init_trim(buf): + return cnt + +def trim(): + return out_buf + +def post_trim(success): + return next_index +``` + +### Custom Mutation + +- `init` (optional): + + This method is called when AFL++ starts up and is used to seed RNG. + +- `fuzz` (required): + + This method performs custom mutations on a given input. It also accepts an + additional test case. + +- `pre_save` (optional): + + For some cases, the format of the mutated data returned from the custom + mutator is not suitable to directly execute the target with this input. + For example, when using libprotobuf-mutator, the data returned is in a + protobuf format which corresponds to a given grammar. In order to execute + the target, the protobuf data must be converted to the plain-text format expected by the target. In such scenarios, the user can define the + `pre_save` function. This function is then transforms the data into the + format expected by the API before executing the target. + + +### Trimming Support + +The generic trimming routines implemented in AFL++ can easily destroy the +structure of complex formats, possibly leading to a point where you have a lot +of test cases in the queue that your Python module cannot process anymore but +your target application still accepts. This is especially the case when your +target can process a part of the input (causing coverage) and then errors out +on the remaining input. + +In such cases, it makes sense to implement a custom trimming routine. The API +consists of multiple methods because after each trimming step, we have to go +back into the C code to check if the coverage bitmap is still the same for the +trimmed input. Here's a quick API description: + +- `init_trim` (optional): + + This method is called at the start of each trimming operation and receives + the initial buffer. It should return the amount of iteration steps possible + on this input (e.g. if your input has n elements and you want to remove them + one by one, return n, if you do a binary search, return log(n), and so on). + + If your trimming algorithm doesn't allow you to determine the amount of + (remaining) steps easily (esp. while running), then you can alternatively + return 1 here and always return 0 in `post_trim` until you are finished and + no steps remain. In that case, returning 1 in `post_trim` will end the + trimming routine. The whole current index/max iterations stuff is only used + to show progress. + +- `trim` (optional) + + This method is called for each trimming operation. It doesn't have any + arguments because we already have the initial buffer from `init_trim` and we + can memorize the current state in global variables. This can also save + reparsing steps for each iteration. It should return the trimmed input + buffer, where the returned data must not exceed the initial input data in + length. Returning anything that is larger than the original data (passed to + `init_trim`) will result in a fatal abort of AFL++. + +- `post_trim` (optional) + + This method is called after each trim operation to inform you if your + trimming step was successful or not (in terms of coverage). If you receive + a failure here, you should reset your input to the last known good state. + In any case, this method must return the next trim iteration index (from 0 + to the maximum amount of steps you returned in `init_trim`). + +Omitting any of three methods will cause the trimming to be disabled and trigger +a fallback to the builtin default trimming routine. + +### Environment Variables + +Optionally, the following environment variables are supported: + +- `AFL_PYTHON_ONLY` + + Disable all other mutation stages. This can prevent broken testcases + (those that your Python module can't work with anymore) to fill up your + queue. Best combined with a custom trimming routine (see below) because + trimming can cause the same test breakage like havoc and splice. + +- `AFL_DEBUG` + + When combined with `AFL_NO_UI`, this causes the C trimming code to emit additional messages about the performance and actions of your custom trimmer. Use this to see if it works :) + +## 3) Usage + +### Prerequisite + +For Python mutator, the python 3 or 2 development package is required. On +Debian/Ubuntu/Kali this can be done: + +```bash +sudo apt install python3-dev +# or +sudo apt install python-dev ``` -$CC -shared -Wall -O3 .c -o .so + +Then, AFL++ can be compiled with Python support. The AFL++ Makefile detects +Python 2 and 3 through `python-config` if it is in the PATH and compiles +`afl-fuzz` with the feature if available. + +Note: for some distributions, you might also need the package `python[23]-apt`. +In case your setup is different, set the necessary variables like this: +`PYTHON_INCLUDE=/path/to/python/include LDFLAGS=-L/path/to/python/lib make`. + +### Custom Mutator Preparation + +For C/C++ mutator, the source code must be compiled as a shared object: +```bash +gcc -shared -Wall -O3 example.c -o example.so ``` -Note: unless AFL_CUSTOM_MUTATOR_ONLY is set, it is a state mutator like any -other, so it will be used for some test cases, and other mutators for others. -Only if AFL_CUSTOM_MUTATOR_ONLY is set the afl_custom_mutator() function will -be called every time it needs to mutate a test case. +### Run -For some cases, the format of the mutated data returned from the custom -mutator is not suitable to directly execute the target with this input. -For example, when using libprotobuf-mutator, the data returned is in a -protobuf format which corresponds to a given grammar. -In order to execute the target, the protobuf data must be converted to the -plain-text format expected by the target. -In such scenarios, the user can define the afl_pre_save_handler() function. -This function is then transforms the data into the format expected by the -API before executing the target. -afl_pre_save_handler is optional and does not have to be implemented if its -functionality is not needed. +C/C++ +```bash +export AFL_CUSTOM_MUTATOR_LIBRARY=/full/path/to/example.so +afl-fuzz /path/to/program +``` + +Python +```bash +export PYTHONPATH=`dirname /full/path/to/example.py` +export AFL_PYTHON_MODULE=example +afl-fuzz /path/to/program +``` -## 2) Example +## 4) Example -A simple example is provided in ../examples/custom_mutators/ +Please see [example.c](../examples/custom_mutators/example.c) and +[example.py](../examples/custom_mutators/example.py) -There is also a libprotobuf example available at [https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator) -Another implementation can be found at [https://github.com/thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator) +## 6) Other Resources +- AFL libprotobuf mutator + - [bruce30262/libprotobuf-mutator_fuzzing_learning](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator) + - [thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator) +- [XML Fuzzing@NullCon 2017](https://www.agarri.fr/docs/XML_Fuzzing-NullCon2017-PUBLIC.pdf) + - [A bug detected by AFL + XML-aware mutators](https://bugs.chromium.org/p/chromium/issues/detail?id=930663) diff --git a/docs/env_variables.md b/docs/env_variables.md index 5214f808..83f5b7c0 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -228,10 +228,10 @@ checks or alter some of the more exotic semantics of the tool: afl-fuzz), setting AFL_PYTHON_MODULE to a Python module can also provide additional mutations. If AFL_CUSTOM_MUTATOR_ONLY is also set, all mutations will solely be - performed with/from the library/Python module. - This feature allows to configure custom mutators which can be very helpful - in e.g. fuzzing XML or other highly flexible structured input. - Please see [custom_mutator.md](custom_mutator.md) or [python_mutators.md](python_mutators.md). + performed with the custom mutator. + This feature allows to configure custom mutators which can be very helpful, + e.g. fuzzing XML or other highly flexible structured input. + Please see [custom_mutator.md](custom_mutator.md). - AFL_FAST_CAL keeps the calibration stage about 2.5x faster (albeit less precise), which can help when starting a session against a slow target. diff --git a/docs/python_mutators.md b/docs/python_mutators.md deleted file mode 100644 index a7e2c7de..00000000 --- a/docs/python_mutators.md +++ /dev/null @@ -1,148 +0,0 @@ -# Adding custom mutators to AFL using Python modules - - This file describes how you can utilize the external Python API to write - your own custom mutation routines. - - Note: This feature is highly experimental. Use at your own risk. - - Implemented by Christian Holler (:decoder) . - - NOTE: Only cPython 2.7, 3.7 and above are supported, although others may work. - Depending on with which version afl-fuzz was compiled against, you must use - python2 or python3 syntax in your scripts! - After a major version upgrade (e.g. 3.7 -> 3.8), a recompilation of afl-fuzz may be needed. - - For an example and a template see ../examples/python_mutators/ - - -## 1) Description and purpose - -While AFLFuzz comes with a good selection of generic deterministic and -non-deterministic mutation operations, it sometimes might make sense to extend -these to implement strategies more specific to the target you are fuzzing. - -For simplicity and in order to allow people without C knowledge to extend -AFLFuzz, I implemented a "Python" stage that can make use of an external -module (written in Python) that implements a custom mutation stage. - -The main motivation behind this is to lower the barrier for people -experimenting with this tool. Hopefully, someone will be able to do useful -things with this extension. - -If you find it useful, have questions or need additional features added to the -interface, feel free to send a mail to . - -See the following information to get a better pictures: - https://www.agarri.fr/docs/XML_Fuzzing-NullCon2017-PUBLIC.pdf - https://bugs.chromium.org/p/chromium/issues/detail?id=930663 - - -## 2) How the Python module looks like - -You can find a simple example in pymodules/example.py including documentation -explaining each function. In the same directory, you can find another simple -module that performs simple mutations. - -Right now, "init" is called at program startup and can be used to perform any -kinds of one-time initializations while "fuzz" is called each time a mutation -is requested. - -There is also optional support for a trimming API, see the section below for -further information about this feature. - - -## 3) How to compile AFLFuzz with Python support - -You must install the python 3 or 2 development package of your Linux -distribution before this will work. On Debian/Ubuntu/Kali this can be done -with either: - apt install python3-dev -or - apt install python-dev -Note that for some distributions you might also need the package python[23]-apt - -A prerequisite for using this mode is to compile AFLFuzz with Python support. - -The AFL++ Makefile detects Python 3 and 2 through `python-config` if is is in the PATH -and compiles afl-fuzz with the feature if available. - -In case your setup is different set the necessary variables like this: -PYTHON_INCLUDE=/path/to/python/include LDFLAGS=-L/path/to/python/lib make - - -## 4) How to run AFLFuzz with your custom module - -You must pass the module name inside the env variable AFL_PYTHON_MODULE. - -In addition, if you are trying to load the module from the local directory, -you must adjust your PYTHONPATH to reflect this circumstance. The following -command should work if you are inside the aflfuzz directory: - -$ AFL_PYTHON_MODULE="pymodules.test" PYTHONPATH=. ./afl-fuzz - -Optionally, the following environment variables are supported: - -AFL_PYTHON_ONLY - Disable all other mutation stages. This can prevent broken - testcases (those that your Python module can't work with - anymore) to fill up your queue. Best combined with a custom - trimming routine (see below) because trimming can cause the - same test breakage like havoc and splice. - -AFL_DEBUG - When combined with AFL_NO_UI, this causes the C trimming code - to emit additional messages about the performance and actions - of your custom Python trimmer. Use this to see if it works :) - - -## 5) Order and statistics - -The Python stage is set to be the first non-deterministic stage (right before -the havoc stage). In the statistics however, it shows up as the third number -under "havoc". That's because I'm lazy and I didn't want to mess with the UI -too much ;) - - -## 6) Trimming support - -The generic trimming routines implemented in AFLFuzz can easily destroy the -structure of complex formats, possibly leading to a point where you have a lot -of testcases in the queue that your Python module cannot process anymore but -your target application still accepts. This is especially the case when your -target can process a part of the input (causing coverage) and then errors out -on the remaining input. - -In such cases, it makes sense to implement a custom trimming routine in Python. -The API consists of multiple methods because after each trimming step, we have -to go back into the C code to check if the coverage bitmap is still the same -for the trimmed input. Here's a quick API description: - -init_trim: This method is called at the start of each trimming operation - and receives the initial buffer. It should return the amount - of iteration steps possible on this input (e.g. if your input - has n elements and you want to remove them one by one, return n, - if you do a binary search, return log(n), and so on...). - - If your trimming algorithm doesn't allow you to determine the - amount of (remaining) steps easily (esp. while running), then you - can alternatively return 1 here and always return 0 in post_trim - until you are finished and no steps remain. In that case, - returning 1 in post_trim will end the trimming routine. The whole - current index/max iterations stuff is only used to show progress. - -trim: This method is called for each trimming operation. It doesn't - have any arguments because we already have the initial buffer - from init_trim and we can memorize the current state in global - variables. This can also save reparsing steps for each iteration. - It should return the trimmed input buffer, where the returned data - must not exceed the initial input data in length. Returning anything - that is larger than the original data (passed to init_trim) will - result in a fatal abort of AFLFuzz. - -post_trim: This method is called after each trim operation to inform you - if your trimming step was successful or not (in terms of coverage). - If you receive a failure here, you should reset your input to the - last known good state. - In any case, this method must return the next trim iteration index - (from 0 to the maximum amount of steps you returned in init_trim). - -Omitting any of the methods will cause Python trimming to be disabled and -trigger a fallback to the builtin default trimming routine. diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index a96ee1d0..2d5a5743 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -194,7 +194,7 @@ static void usage(u8* argv0, int more_help) { "use \"-hh\".\n\n"); #ifdef USE_PYTHON - SAYF("Compiled with %s module support, see docs/python_mutators.md\n", + SAYF("Compiled with %s module support, see docs/custom_mutator.md\n", (char*)PYTHON_VERSION); #endif -- cgit 1.4.1