From 7b59e05600aff4e9deb3110887d6ad49838f02b0 Mon Sep 17 00:00:00 2001 From: h1994st Date: Mon, 2 Mar 2020 15:27:29 -0500 Subject: Add new APIs for the custom mutator --- src/afl-fuzz-python.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/afl-fuzz-python.c') diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 42286527..be86ebba 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -55,6 +55,8 @@ int init_py() { u8 py_notrim = 0, py_idx; py_functions[PY_FUNC_INIT] = PyObject_GetAttrString(py_module, "init"); py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "fuzz"); + py_functions[PY_FUNC_PRE_SAVE] = + PyObject_GetAttrString(py_module, "pre_save"); py_functions[PY_FUNC_INIT_TRIM] = PyObject_GetAttrString(py_module, "init_trim"); py_functions[PY_FUNC_POST_TRIM] = -- cgit 1.4.1 From 7862416844a2636d37754b8b2175dbd97494771f Mon Sep 17 00:00:00 2001 From: h1994st Date: Mon, 2 Mar 2020 19:29:41 -0500 Subject: Uniform API for both Python and custom mutator --- include/afl-fuzz.h | 67 +++++++---- src/afl-fuzz-globals.c | 5 - src/afl-fuzz-mutators.c | 207 ++++++++++++++++++++++++++++++---- src/afl-fuzz-one.c | 9 +- src/afl-fuzz-python.c | 294 ++++++++++++++---------------------------------- src/afl-fuzz-run.c | 9 +- src/afl-fuzz-stats.c | 2 +- 7 files changed, 322 insertions(+), 271 deletions(-) (limited to 'src/afl-fuzz-python.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index b8b6d5e3..e6aaee68 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -472,12 +472,12 @@ struct custom_mutator { * * (Optional) */ - u32 (*afl_custom_init)(void); + void (*afl_custom_init)(unsigned int seed); /** * Perform custom mutations on a given input * - * (Required) + * (Optional for now. Required in the future) * * @param[in] data Input data to be mutated * @param[in] size Size of input data @@ -498,46 +498,67 @@ struct custom_mutator { * (Optional) If this functionality is not needed, simply don't define this * function. * - * @param[in] data Buffer containing the test case to be executed. - * @param[in] size Size of the test case. + * @param[in] data Buffer containing the test case to be executed + * @param[in] size Size of the test case * @param[out] new_data Buffer to store the test case after processing - * @return Size of data after processing. + * @return Size of data after processing */ size_t (*afl_custom_pre_save)(u8* data, size_t size, u8** new_data); /** - * TODO: figure out what `trim` is + * This method is called at the start of each trimming operation and receives + * the initial buffer. It should return the amount of iteration steps possible + * on this input (e.g. if your input has n elements and you want to remove + * them one by one, return n, if you do a binary search, return log(n), + * and so on...). + * + * If your trimming algorithm doesn't allow you to determine the amount of + * (remaining) steps easily (esp. while running), then you can alternatively + * return 1 here and always return 0 in post_trim until you are finished and + * no steps remain. In that case, returning 1 in post_trim will end the + * trimming routine. The whole current index/max iterations stuff is only used + * to show progress. * * (Optional) + * + * @param data Buffer containing the test case + * @param size Size of the test case + * @return The amount of possible iteration steps to trim the input */ - u32 (*afl_custom_init_trim)(u8*, size_t); + u32 (*afl_custom_init_trim)(u8* data, size_t size); /** - * TODO: figure out how `trim` works + * This method is called for each trimming operation. It doesn't have any + * arguments because we already have the initial buffer from init_trim and we + * can memorize the current state in global variables. This can also save + * reparsing steps for each iteration. It should return the trimmed input + * buffer, where the returned data must not exceed the initial input data in + * length. Returning anything that is larger than the original data (passed + * to init_trim) will result in a fatal abort of AFLFuzz. * * (Optional) * - * @param[out] ret (TODO: finish here) - * @param[out] ret_len (TODO: finish here) + * @param[out] ret Buffer containing the trimmed test case + * @param[out] ret_len Size of the trimmed test case */ void (*afl_custom_trim)(u8** ret, size_t* ret_len); /** - * A post-processing function for the last trim operation. + * This method is called after each trim operation to inform you if your + * trimming step was successful or not (in terms of coverage). If you receive + * a failure here, you should reset your input to the last known good state. * * (Optional) * * @param success Indicates if the last trim operation was successful. + * @return The next trim iteration index (from 0 to the maximum amount of + * steps returned in init_trim) */ u32 (*afl_custom_post_trim)(u8 success); }; extern struct custom_mutator* mutator; -size_t (*custom_mutator)(u8* data, size_t size, u8* mutated_out, - size_t max_size, unsigned int seed); -size_t (*pre_save_handler)(u8* data, size_t size, u8** new_data); - /* Interesting values, as per config.h */ extern s8 interesting_8[INTERESTING_8_LEN]; @@ -598,17 +619,19 @@ void setup_custom_mutator(void); void destroy_custom_mutator(void); void load_custom_mutator(const char*); void load_custom_mutator_py(const char*); +u8 trim_case_custom(char** argv, struct queue_entry* q, u8* in_buf); /* Python */ #ifdef USE_PYTHON -int init_py(); -void finalize_py(); -void fuzz_py(char*, size_t, char*, size_t, char**, size_t*); +int init_py_module(u8*); +void finalize_py_module(); + +void init_py(unsigned int seed); +void fuzz_py(char*, size_t, char*, size_t, char**, size_t*); size_t pre_save_py(u8* data, size_t size, u8** new_data); -u32 init_trim_py(char*, size_t); -u32 post_trim_py(char); -void trim_py(char**, size_t*); -u8 trim_case_python(char**, struct queue_entry*, u8*); +u32 init_trim_py(u8*, size_t); +u32 post_trim_py(u8); +void trim_py(u8**, size_t*); #endif /* Queue */ diff --git a/src/afl-fuzz-globals.c b/src/afl-fuzz-globals.c index 1fd4b26d..a054499f 100644 --- a/src/afl-fuzz-globals.c +++ b/src/afl-fuzz-globals.c @@ -259,11 +259,6 @@ s32 cmplog_child_pid, cmplog_forksrv_pid; /* Custom mutator */ struct custom_mutator* mutator; -/* hooks for the custom mutator function */ -size_t (*custom_mutator)(u8 *data, size_t size, u8 *mutated_out, - size_t max_size, unsigned int seed); -size_t (*pre_save_handler)(u8 *data, size_t size, u8 **new_data); - /* Interesting values, as per config.h */ s8 interesting_8[] = {INTERESTING_8}; diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 0e77a690..22ca1384 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -42,11 +42,22 @@ void setup_custom_mutator(void) { } #ifdef USE_PYTHON - if (init_py()) FATAL("Failed to initialize Python module"); + u8* module_name = getenv("AFL_PYTHON_MODULE"); - // u8* module_name = getenv("AFL_PYTHON_MODULE"); - // if (py_module && module_name) - // load_custom_mutator_py(module_name); + if (module_name) { + + if (limit_time_sig) + FATAL( + "MOpt and Python mutator are mutually exclusive. We accept pull " + "requests that integrates MOpt with the optional mutators " + "(custom/radamsa/redquenn/...)."); + + if (init_py_module(module_name)) + FATAL("Failed to initialize Python module"); + + load_custom_mutator_py(module_name); + + } #else if (getenv("AFL_PYTHON_MODULE")) FATAL("Your AFL binary was built without Python support"); @@ -62,7 +73,7 @@ void destroy_custom_mutator(void) { else { /* Python mutator */ #ifdef USE_PYTHON - finalize_py(); + finalize_py_module(); #endif } @@ -80,11 +91,11 @@ void load_custom_mutator(const char* fn) { ACTF("Loading custom mutator library from '%s'...", fn); dh = dlopen(fn, RTLD_NOW); - if (!mutator->dh) FATAL("%s", dlerror()); + if (!dh) FATAL("%s", dlerror()); mutator->dh = dh; /* Mutator */ - /* "afl_custom_init", optional */ + /* "afl_custom_init", optional for backward compatibility */ mutator->afl_custom_init = dlsym(dh, "afl_custom_init"); if (!mutator->afl_custom_init) WARNF("Symbol 'afl_custom_init' not found."); @@ -92,13 +103,14 @@ void load_custom_mutator(const char* fn) { /* "afl_custom_fuzz" or "afl_custom_mutator", required */ mutator->afl_custom_fuzz = dlsym(dh, "afl_custom_fuzz"); if (!mutator->afl_custom_fuzz) { + /* Try "afl_custom_mutator" for backward compatibility */ WARNF("Symbol 'afl_custom_fuzz' not found. Try 'afl_custom_mutator'."); mutator->afl_custom_fuzz = dlsym(dh, "afl_custom_mutator"); - if (!mutator->afl_custom_fuzz) { + if (!mutator->afl_custom_fuzz) FATAL("Symbol 'afl_custom_mutator' not found."); - } + } /* "afl_custom_pre_save", optional */ @@ -106,6 +118,7 @@ void load_custom_mutator(const char* fn) { if (!mutator->afl_custom_pre_save) WARNF("Symbol 'afl_custom_pre_save' not found."); + u8 notrim = 0; /* "afl_custom_init_trim", optional */ mutator->afl_custom_init_trim = dlsym(dh, "afl_custom_init_trim"); if (!mutator->afl_custom_init_trim) @@ -121,29 +134,177 @@ void load_custom_mutator(const char* fn) { if (!mutator->afl_custom_post_trim) WARNF("Symbol 'afl_custom_post_trim' not found."); + if (notrim) { + + mutator->afl_custom_init_trim = NULL; + mutator->afl_custom_trim = NULL; + mutator->afl_custom_post_trim = NULL; + WARNF( + "Custom mutator does not implement all three trim APIs, standard " + "trimming will be used."); + + } + OKF("Custom mutator '%s' installed successfully.", fn); /* Initialize the custom mutator */ if (mutator->afl_custom_init) - mutator->afl_custom_init(); + mutator->afl_custom_init(UR(0xFFFFFFFF)); + +} + +u8 trim_case_custom(char** argv, struct queue_entry* q, u8* in_buf) { + + static u8 tmp[64]; + static u8 clean_trace[MAP_SIZE]; + + u8 needs_write = 0, fault = 0; + u32 trim_exec = 0; + u32 orig_len = q->len; + + stage_name = tmp; + bytes_trim_in += q->len; + + /* Initialize trimming in the custom mutator */ + stage_cur = 0; + stage_max = mutator->afl_custom_init_trim(in_buf, q->len); + + if (not_on_tty && debug) + SAYF("[Custom Trimming] START: Max %d iterations, %u bytes", stage_max, + q->len); + + while (stage_cur < stage_max) { + + sprintf(tmp, "ptrim %s", DI(trim_exec)); + + u32 cksum; + + u8* retbuf = NULL; + size_t retlen = 0; + + mutator->afl_custom_trim(&retbuf, &retlen); + + if (retlen > orig_len) + FATAL( + "Trimmed data returned by custom mutator is larger than original " + "data"); + + write_to_testcase(retbuf, retlen); + + fault = run_target(argv, exec_tmout); + ++trim_execs; + + if (stop_soon || fault == FAULT_ERROR) { + + free(retbuf); + goto abort_trimming; + + } + + cksum = hash32(trace_bits, MAP_SIZE, HASH_CONST); + + if (cksum == q->exec_cksum) { + + q->len = retlen; + memcpy(in_buf, retbuf, retlen); + + /* Let's save a clean trace, which will be needed by + update_bitmap_score once we're done with the trimming stuff. */ + + if (!needs_write) { + + needs_write = 1; + memcpy(clean_trace, trace_bits, MAP_SIZE); + + } + + /* Tell the custom mutator that the trimming was successful */ + stage_cur = mutator->afl_custom_post_trim(1); + + if (not_on_tty && debug) + SAYF("[Custom Trimming] SUCCESS: %d/%d iterations (now at %u bytes)", + stage_cur, stage_max, q->len); + + } else { + + /* Tell the custom mutator that the trimming was unsuccessful */ + stage_cur = mutator->afl_custom_post_trim(0); + if (not_on_tty && debug) + SAYF("[Custom Trimming] FAILURE: %d/%d iterations", stage_cur, + stage_max); + + } + + free(retbuf); + + /* Since this can be slow, update the screen every now and then. */ + + if (!(trim_exec++ % stats_update_freq)) show_stats(); + + } + + if (not_on_tty && debug) + SAYF("[Custom Trimming] DONE: %u bytes -> %u bytes", orig_len, q->len); + + /* If we have made changes to in_buf, we also need to update the on-disk + version of the test case. */ + + if (needs_write) { + + s32 fd; + + unlink(q->fname); /* ignore errors */ + + fd = open(q->fname, O_WRONLY | O_CREAT | O_EXCL, 0600); + + if (fd < 0) PFATAL("Unable to create '%s'", q->fname); + + ck_write(fd, in_buf, q->len, q->fname); + close(fd); + + memcpy(trace_bits, clean_trace, MAP_SIZE); + update_bitmap_score(q); + + } + +abort_trimming: + + bytes_trim_out += q->len; + return fault; } -// void load_custom_mutator_py(const char* module_name) { +void load_custom_mutator_py(const char* module_name) { -// mutator = ck_alloc(sizeof(struct custom_mutator)); + mutator = ck_alloc(sizeof(struct custom_mutator)); -// mutator->name = module_name; -// ACTF("Loading Python mutator library from '%s'...", module_name); + mutator->name = module_name; + ACTF("Loading Python mutator library from '%s'...", module_name); -// /* Initialize of the Python mutator has been invoked in "init_py()" */ -// mutator->afl_custom_init = NULL; -// mutator->afl_custom_fuzz = fuzz_py; -// mutator->afl_custom_pre_save = pre_save_py; -// mutator->afl_custom_init_trim = init_trim_py; -// mutator->afl_custom_trim = trim_py; -// mutator->afl_custom_post_trim = post_trim_py; + /* TODO: unify "init" and "fuzz" */ + if (py_functions[PY_FUNC_INIT]) + mutator->afl_custom_init = init_py; -// OKF("Python mutator '%s' installed successfully.", module_name); + /* "afl_custom_fuzz" should not be NULL, but the interface of Python mutator + is quite different from the custom mutator. */ + mutator->afl_custom_fuzz = NULL; -// } + if (py_functions[PY_FUNC_PRE_SAVE]) + mutator->afl_custom_pre_save = pre_save_py; + + if (py_functions[PY_FUNC_INIT_TRIM]) + mutator->afl_custom_init_trim = init_trim_py; + + if (py_functions[PY_FUNC_POST_TRIM]) + mutator->afl_custom_post_trim = post_trim_py; + + if (py_functions[PY_FUNC_TRIM]) + mutator->afl_custom_trim = trim_py; + + OKF("Python mutator '%s' installed successfully.", module_name); + + /* Initialize the custom mutator */ + if (mutator->afl_custom_init) + mutator->afl_custom_init(UR(0xFFFFFFFF)); + +} diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index f1efe2df..24ea76f3 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -449,7 +449,7 @@ u8 fuzz_one_original(char** argv) { * TRIMMING * ************/ - if (!dumb_mode && !queue_cur->trim_done && !custom_mutator && !disable_trim) { + if (!dumb_mode && !queue_cur->trim_done && !disable_trim) { u8 res = trim_case(argv, queue_cur, in_buf); @@ -484,7 +484,7 @@ u8 fuzz_one_original(char** argv) { // custom_stage: // not used - yet - if (custom_mutator) { + if (mutator->afl_custom_fuzz) { stage_short = "custom"; stage_name = "custom mutator"; @@ -499,8 +499,9 @@ u8 fuzz_one_original(char** argv) { for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) { size_t orig_size = (size_t)len; - size_t mutated_size = custom_mutator(in_buf, orig_size, mutated_buf, - max_seed_size, UR(UINT32_MAX)); + size_t mutated_size = mutator->afl_custom_fuzz(in_buf, orig_size, + mutated_buf, max_seed_size, + UR(UINT32_MAX)); if (mutated_size > 0) { out_buf = ck_realloc(out_buf, mutated_size); diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index be86ebba..da478cc2 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -28,116 +28,73 @@ /* Python stuff */ #ifdef USE_PYTHON -int init_py() { +int init_py_module(u8* module_name) { - Py_Initialize(); - u8* module_name = getenv("AFL_PYTHON_MODULE"); - - if (module_name) { - - if (limit_time_sig) - FATAL( - "MOpt and Python mutator are mutually exclusive. We accept pull " - "requests that integrates MOpt with the optional mutators " - "(custom/radamsa/redquenn/...)."); + if (!module_name) return 1; #if PY_MAJOR_VERSION >= 3 - PyObject* py_name = PyUnicode_FromString(module_name); + PyObject* py_name = PyUnicode_FromString(module_name); #else - PyObject* py_name = PyString_FromString(module_name); + PyObject* py_name = PyString_FromString(module_name); #endif - py_module = PyImport_Import(py_name); - Py_DECREF(py_name); + py_module = PyImport_Import(py_name); + Py_DECREF(py_name); - if (py_module != NULL) { - - u8 py_notrim = 0, py_idx; - py_functions[PY_FUNC_INIT] = PyObject_GetAttrString(py_module, "init"); - py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "fuzz"); - py_functions[PY_FUNC_PRE_SAVE] = - PyObject_GetAttrString(py_module, "pre_save"); - py_functions[PY_FUNC_INIT_TRIM] = - PyObject_GetAttrString(py_module, "init_trim"); - py_functions[PY_FUNC_POST_TRIM] = - PyObject_GetAttrString(py_module, "post_trim"); - py_functions[PY_FUNC_TRIM] = PyObject_GetAttrString(py_module, "trim"); + if (py_module != NULL) { - for (py_idx = 0; py_idx < PY_FUNC_COUNT; ++py_idx) { + u8 py_notrim = 0, py_idx; + py_functions[PY_FUNC_INIT] = PyObject_GetAttrString(py_module, "init"); + py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "fuzz"); + py_functions[PY_FUNC_PRE_SAVE] = + PyObject_GetAttrString(py_module, "pre_save"); + py_functions[PY_FUNC_INIT_TRIM] = + PyObject_GetAttrString(py_module, "init_trim"); + py_functions[PY_FUNC_POST_TRIM] = + PyObject_GetAttrString(py_module, "post_trim"); + py_functions[PY_FUNC_TRIM] = PyObject_GetAttrString(py_module, "trim"); - if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { + for (py_idx = 0; py_idx < PY_FUNC_COUNT; ++py_idx) { - if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { + if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { - // Implementing the trim API is optional for now - if (PyErr_Occurred()) PyErr_Print(); - py_notrim = 1; + if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { - } else { + // Implementing the trim API is optional for now + if (PyErr_Occurred()) PyErr_Print(); + py_notrim = 1; - if (PyErr_Occurred()) PyErr_Print(); - fprintf(stderr, - "Cannot find/call function with index %d in external " - "Python module.\n", - py_idx); - return 1; + } else { - } + if (PyErr_Occurred()) PyErr_Print(); + fprintf(stderr, + "Cannot find/call function with index %d in external " + "Python module.\n", + py_idx); + return 1; } } - if (py_notrim) { - - py_functions[PY_FUNC_INIT_TRIM] = NULL; - py_functions[PY_FUNC_POST_TRIM] = NULL; - py_functions[PY_FUNC_TRIM] = NULL; - WARNF( - "Python module does not implement trim API, standard trimming will " - "be used."); - - } - - PyObject *py_args, *py_value; - - /* Provide the init function a seed for the Python RNG */ - py_args = PyTuple_New(1); -#if PY_MAJOR_VERSION >= 3 - py_value = PyLong_FromLong(UR(0xFFFFFFFF)); -#else - py_value = PyInt_FromLong(UR(0xFFFFFFFF)); -#endif - - if (!py_value) { - - Py_DECREF(py_args); - fprintf(stderr, "Cannot convert argument\n"); - return 1; - - } - - PyTuple_SetItem(py_args, 0, py_value); - - py_value = PyObject_CallObject(py_functions[PY_FUNC_INIT], py_args); - - Py_DECREF(py_args); - - if (py_value == NULL) { + } - PyErr_Print(); - fprintf(stderr, "Call failed\n"); - return 1; + if (py_notrim) { - } + py_functions[PY_FUNC_INIT_TRIM] = NULL; + py_functions[PY_FUNC_POST_TRIM] = NULL; + py_functions[PY_FUNC_TRIM] = NULL; + WARNF( + "Python module does not implement trim API, standard trimming will " + "be used."); - } else { + } - PyErr_Print(); - fprintf(stderr, "Failed to load \"%s\"\n", module_name); - return 1; + } else { - } + PyErr_Print(); + fprintf(stderr, "Failed to load \"%s\"\n", module_name); + return 1; } @@ -145,7 +102,7 @@ int init_py() { } -void finalize_py() { +void finalize_py_module() { if (py_module != NULL) { @@ -213,7 +170,43 @@ void fuzz_py(char* buf, size_t buflen, char* add_buf, size_t add_buflen, } -u32 init_trim_py(char* buf, size_t buflen) { +size_t pre_save_py(u8* data, size_t size, u8** new_data) { + + size_t new_size; + PyObject *py_args, *py_value; + py_args = PyTuple_New(2); + py_value = PyByteArray_FromStringAndSize(data, size); + if (!py_value) { + + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + + } + + PyTuple_SetItem(py_args, 0, py_value); + + py_value = PyObject_CallObject(py_functions[PY_FUNC_PRE_SAVE], py_args); + + Py_DECREF(py_args); + + if (py_value != NULL) { + + new_size = PyByteArray_Size(py_value); + *new_data = malloc(new_size); + memcpy(*new_data, PyByteArray_AsString(py_value), new_size); + Py_DECREF(py_value); + return new_size; + + } else { + + PyErr_Print(); + FATAL("Call failed"); + + } + +} + +u32 init_trim_py(u8* buf, size_t buflen) { PyObject *py_args, *py_value; @@ -250,7 +243,7 @@ u32 init_trim_py(char* buf, size_t buflen) { } -u32 post_trim_py(char success) { +u32 post_trim_py(u8 success) { PyObject *py_args, *py_value; @@ -288,7 +281,7 @@ u32 post_trim_py(char success) { } -void trim_py(char** ret, size_t* retlen) { +void trim_py(u8** ret, size_t* retlen) { PyObject *py_args, *py_value; @@ -312,126 +305,5 @@ void trim_py(char** ret, size_t* retlen) { } -u8 trim_case_python(char** argv, struct queue_entry* q, u8* in_buf) { - - static u8 tmp[64]; - static u8 clean_trace[MAP_SIZE]; - - u8 needs_write = 0, fault = 0; - u32 trim_exec = 0; - u32 orig_len = q->len; - - stage_name = tmp; - bytes_trim_in += q->len; - - /* Initialize trimming in the Python module */ - stage_cur = 0; - stage_max = init_trim_py(in_buf, q->len); - - if (not_on_tty && debug) - SAYF("[Python Trimming] START: Max %d iterations, %u bytes", stage_max, - q->len); - - while (stage_cur < stage_max) { - - sprintf(tmp, "ptrim %s", DI(trim_exec)); - - u32 cksum; - - char* retbuf = NULL; - size_t retlen = 0; - - trim_py(&retbuf, &retlen); - - if (retlen > orig_len) - FATAL( - "Trimmed data returned by Python module is larger than original " - "data"); - - write_to_testcase(retbuf, retlen); - - fault = run_target(argv, exec_tmout); - ++trim_execs; - - if (stop_soon || fault == FAULT_ERROR) { - - free(retbuf); - goto abort_trimming; - - } - - cksum = hash32(trace_bits, MAP_SIZE, HASH_CONST); - - if (cksum == q->exec_cksum) { - - q->len = retlen; - memcpy(in_buf, retbuf, retlen); - - /* Let's save a clean trace, which will be needed by - update_bitmap_score once we're done with the trimming stuff. */ - - if (!needs_write) { - - needs_write = 1; - memcpy(clean_trace, trace_bits, MAP_SIZE); - - } - - /* Tell the Python module that the trimming was successful */ - stage_cur = post_trim_py(1); - - if (not_on_tty && debug) - SAYF("[Python Trimming] SUCCESS: %d/%d iterations (now at %u bytes)", - stage_cur, stage_max, q->len); - - } else { - - /* Tell the Python module that the trimming was unsuccessful */ - stage_cur = post_trim_py(0); - if (not_on_tty && debug) - SAYF("[Python Trimming] FAILURE: %d/%d iterations", stage_cur, - stage_max); - - } - - free(retbuf); - - /* Since this can be slow, update the screen every now and then. */ - - if (!(trim_exec++ % stats_update_freq)) show_stats(); - - } - - if (not_on_tty && debug) - SAYF("[Python Trimming] DONE: %u bytes -> %u bytes", orig_len, q->len); - - /* If we have made changes to in_buf, we also need to update the on-disk - version of the test case. */ - - if (needs_write) { - - s32 fd; - - unlink(q->fname); /* ignore errors */ - - fd = open(q->fname, O_WRONLY | O_CREAT | O_EXCL, 0600); - - if (fd < 0) PFATAL("Unable to create '%s'", q->fname); - - ck_write(fd, in_buf, q->len, q->fname); - close(fd); - - memcpy(trace_bits, clean_trace, MAP_SIZE); - update_bitmap_score(q); - - } - -abort_trimming: - - bytes_trim_out += q->len; - return fault; - -} - #endif /* USE_PYTHON */ diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 95c2c5d4..a1d10387 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -309,10 +309,10 @@ void write_to_testcase(void* mem, u32 len) { lseek(fd, 0, SEEK_SET); - if (pre_save_handler) { + if (mutator->afl_custom_pre_save) { u8* new_data; - size_t new_size = pre_save_handler(mem, len, &new_data); + size_t new_size = mutator->afl_custom_pre_save(mem, len, &new_data); ck_write(fd, new_data, new_size, out_file); } else { @@ -678,9 +678,8 @@ void sync_fuzzers(char** argv) { u8 trim_case(char** argv, struct queue_entry* q, u8* in_buf) { -#ifdef USE_PYTHON - if (py_functions[PY_FUNC_TRIM]) return trim_case_python(argv, q, in_buf); -#endif + /* Custom mutator trimmer */ + if (mutator->afl_custom_trim) return trim_case_custom(argv, q, in_buf); static u8 tmp[64]; static u8 clean_trace[MAP_SIZE]; diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index c1aa8315..1b763c01 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -655,7 +655,7 @@ void show_stats(void) { } - if (custom_mutator) { + if (mutator) { sprintf(tmp, "%s/%s", DI(stage_finds[STAGE_CUSTOM_MUTATOR]), DI(stage_cycles[STAGE_CUSTOM_MUTATOR])); -- cgit 1.4.1 From b2a2b0fc212909df0806abecdd5d64833ae3d3e1 Mon Sep 17 00:00:00 2001 From: h1994st Date: Mon, 2 Mar 2020 19:30:05 -0500 Subject: Add initialization funcation wrapper for Python mutator --- src/afl-fuzz-python.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'src/afl-fuzz-python.c') diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index da478cc2..30156fa6 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -118,6 +118,40 @@ void finalize_py_module() { } +void init_py(unsigned int seed) { + PyObject *py_args, *py_value; + + /* Provide the init function a seed for the Python RNG */ + py_args = PyTuple_New(1); +#if PY_MAJOR_VERSION >= 3 + py_value = PyLong_FromLong(seed); +#else + py_value = PyInt_FromLong(seed); +#endif + + if (!py_value) { + + Py_DECREF(py_args); + fprintf(stderr, "Cannot convert argument\n"); + return; + + } + + PyTuple_SetItem(py_args, 0, py_value); + + py_value = PyObject_CallObject(py_functions[PY_FUNC_INIT], py_args); + + Py_DECREF(py_args); + + if (py_value == NULL) { + + PyErr_Print(); + fprintf(stderr, "Call failed\n"); + return; + + } +} + void fuzz_py(char* buf, size_t buflen, char* add_buf, size_t add_buflen, char** ret, size_t* retlen) { -- cgit 1.4.1 From 90506479e7de57c97d97958c61b2513009687d90 Mon Sep 17 00:00:00 2001 From: h1994st Date: Mon, 2 Mar 2020 21:30:10 -0500 Subject: Refactoring `fuzz_py` API --- include/afl-fuzz.h | 4 ++- src/afl-fuzz-mutators.c | 1 - src/afl-fuzz-one.c | 2 +- src/afl-fuzz-python.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++-- src/afl-fuzz.c | 7 +++-- 5 files changed, 85 insertions(+), 9 deletions(-) (limited to 'src/afl-fuzz-python.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index e6aaee68..27b22082 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -627,7 +627,9 @@ int init_py_module(u8*); void finalize_py_module(); void init_py(unsigned int seed); -void fuzz_py(char*, size_t, char*, size_t, char**, size_t*); +/* TODO: unify fuzz interface for custom mutator and Python mutator */ +size_t fuzz_py(u8*, size_t, u8*, size_t, unsigned int); +void fuzz_py_original(char*, size_t, char*, size_t, char**, size_t*); size_t pre_save_py(u8* data, size_t size, u8** new_data); u32 init_trim_py(u8*, size_t); u32 post_trim_py(u8); diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 22ca1384..9365d487 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -281,7 +281,6 @@ void load_custom_mutator_py(const char* module_name) { mutator->name = module_name; ACTF("Loading Python mutator library from '%s'...", module_name); - /* TODO: unify "init" and "fuzz" */ if (py_functions[PY_FUNC_INIT]) mutator->afl_custom_init = init_py; diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 24ea76f3..1e6dd45d 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -1647,7 +1647,7 @@ python_stage: ck_read(fd, new_buf, target->len, target->fname); close(fd); - fuzz_py(out_buf, len, new_buf, target->len, &retbuf, &retlen); + fuzz_py_original(out_buf, len, new_buf, target->len, &retbuf, &retlen); ck_free(new_buf); diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 30156fa6..c8caa4c1 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -32,6 +32,8 @@ int init_py_module(u8* module_name) { if (!module_name) return 1; + Py_Initialize(); + #if PY_MAJOR_VERSION >= 3 PyObject* py_name = PyUnicode_FromString(module_name); #else @@ -58,7 +60,12 @@ int init_py_module(u8* module_name) { if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { - if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { + if (py_idx == PY_FUNC_PRE_SAVE) { + + // Implenting the pre_save API is optional for now + if (PyErr_Occurred()) PyErr_Print(); + + } else if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { // Implementing the trim API is optional for now if (PyErr_Occurred()) PyErr_Print(); @@ -152,8 +159,9 @@ void init_py(unsigned int seed) { } } -void fuzz_py(char* buf, size_t buflen, char* add_buf, size_t add_buflen, - char** ret, size_t* retlen) { +void fuzz_py_original(char* buf, size_t buflen, + char* add_buf, size_t add_buflen, + char** ret, size_t* retlen) { if (py_module != NULL) { @@ -204,6 +212,72 @@ void fuzz_py(char* buf, size_t buflen, char* add_buf, size_t add_buflen, } +size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, + unsigned int seed) { + + size_t out_size; + PyObject *py_args, *py_value; + py_args = PyTuple_New(3); + + py_value = PyByteArray_FromStringAndSize(data, size); + if (!py_value) { + + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + + } + + PyTuple_SetItem(py_args, 0, py_value); + +#if PY_MAJOR_VERSION >= 3 + py_value = PyLong_FromLong(max_size); +#else + py_value = PyInt_FromLong(max_size); +#endif + if (!py_value) { + + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + + } + + PyTuple_SetItem(py_args, 1, py_value); + +#if PY_MAJOR_VERSION >= 3 + py_value = PyLong_FromLong(seed); +#else + py_value = PyInt_FromLong(seed); +#endif + if (!py_value) { + + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + + } + + PyTuple_SetItem(py_args, 2, py_value); + + py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args); + + Py_DECREF(py_args); + + if (py_value != NULL) { + + out_size = PyByteArray_Size(py_value); + memcpy(mutated_out, PyByteArray_AsString(py_value), out_size); + Py_DECREF(py_value); + + return out_size; + + } else { + + PyErr_Print(); + FATAL("Call failed"); + + } + +} + size_t pre_save_py(u8* data, size_t size, u8** new_data) { size_t new_size; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 2f0043ab..d329a20e 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -752,9 +752,9 @@ int main(int argc, char** argv, char** envp) { if (get_afl_env("AFL_FAST_CAL")) fast_cal = 1; if (get_afl_env("AFL_AUTORESUME")) { - + autoresume = 1; - if (in_place_resume) + if (in_place_resume) SAYF("AFL_AUTORESUME has no effect for '-i -'"); } @@ -862,7 +862,6 @@ int main(int argc, char** argv, char** envp) { check_cpu_governor(); setup_post(); - setup_custom_mutator(); setup_shm(dumb_mode); if (!in_bitmap) memset(virgin_bits, 255, MAP_SIZE); @@ -873,6 +872,8 @@ int main(int argc, char** argv, char** envp) { setup_dirs_fds(); + setup_custom_mutator(); + setup_cmdline_file(argv + optind); read_testcases(); -- cgit 1.4.1 From df465216583afcc0e65e4468e6383afd7a688ddc Mon Sep 17 00:00:00 2001 From: h1994st Date: Tue, 3 Mar 2020 19:48:13 -0500 Subject: Finish refactoring APIs for the custom mutator and Python module - Remove AFL_PYTHON_ONLY (env) and python_only (variable) - Unify fuzz API of the custom mutator and Python module - Merge the custom mutator into the old python_stage, which is now renamed to custom_mutator_stage --- .gitignore | 2 + docs/custom_mutator.md | 2 +- docs/env_variables.md | 34 +++++++------- include/afl-fuzz.h | 50 +++++++++++--------- include/envs.h | 2 +- src/afl-fuzz-globals.c | 3 +- src/afl-fuzz-mutators.c | 4 +- src/afl-fuzz-one.c | 121 ++++++++++++------------------------------------ src/afl-fuzz-python.c | 108 +++++++++++------------------------------- src/afl-fuzz-run.c | 3 +- src/afl-fuzz.c | 18 ++----- 11 files changed, 115 insertions(+), 232 deletions(-) (limited to 'src/afl-fuzz-python.c') diff --git a/.gitignore b/.gitignore index 4032b826..bdd342c4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,9 +18,11 @@ afl-qemu-trace afl-showmap afl-tmin afl-analyze.8 +afl-as.8 afl-clang-fast++.8 afl-clang-fast.8 afl-cmin.8 +afl-cmin.bash.8 afl-fuzz.8 afl-gcc.8 afl-gcc-fast.8 diff --git a/docs/custom_mutator.md b/docs/custom_mutator.md index dff32c1d..68e27de7 100644 --- a/docs/custom_mutator.md +++ b/docs/custom_mutator.md @@ -13,7 +13,7 @@ a given grammar. The custom mutator library is passed to afl-fuzz via the AFL_CUSTOM_MUTATOR_LIBRARY environment variable. The library must export -the afl_custom_mutator() function and must be compiled as a shared object. +the afl_custom_fuzz() function and must be compiled as a shared object. For example: ``` $CC -shared -Wall -O3 .c -o .so diff --git a/docs/env_variables.md b/docs/env_variables.md index 527f1c1b..5214f808 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -104,7 +104,7 @@ Then there are a few specific features that are only available in llvm_mode: - Setting AFL_LLVM_LAF_SPLIT_COMPARES will split all floating point and 64, 32 and 16 bit integer CMP instructions - See llvm_mode/README.laf-intel.md for more information. + See llvm_mode/README.laf-intel.md for more information. ### WHITELIST @@ -192,7 +192,7 @@ checks or alter some of the more exotic semantics of the tool: deciding if a particular test case is a "hang". The default is 1 second or the value of the -t parameter, whichever is larger. Dialing the value down can be useful if you are very concerned about slow inputs, or if you - don't want AFL to spend too much time classifying that stuff and just + don't want AFL to spend too much time classifying that stuff and just rapidly put all timeouts in that bin. - AFL_NO_ARITH causes AFL to skip most of the deterministic arithmetics. @@ -223,15 +223,15 @@ checks or alter some of the more exotic semantics of the tool: for more. - Setting AFL_CUSTOM_MUTATOR_LIBRARY to a shared library with - afl_custom_mutator() creates additional mutations through this library. + afl_custom_fuzz() creates additional mutations through this library. + If afl-fuzz is compiled with Python (which is autodetected during builing + afl-fuzz), setting AFL_PYTHON_MODULE to a Python module can also provide + additional mutations. If AFL_CUSTOM_MUTATOR_ONLY is also set, all mutations will solely be - performed with/from the library. See [custom_mutator.md](custom_mutator.md) - - - For AFL_PYTHON_MODULE and AFL_PYTHON_ONLY - they require afl-fuzz to - be compiled with Python (which is autodetected during builing afl-fuzz). - Please see [python_mutators.md](python_mutators.md). + performed with/from the library/Python module. This feature allows to configure custom mutators which can be very helpful in e.g. fuzzing XML or other highly flexible structured input. + Please see [custom_mutator.md](custom_mutator.md) or [python_mutators.md](python_mutators.md). - AFL_FAST_CAL keeps the calibration stage about 2.5x faster (albeit less precise), which can help when starting a session against a slow target. @@ -283,7 +283,7 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - Setting AFL_INST_LIBS causes the translator to also instrument the code inside any dynamically linked libraries (notably including glibc). - + - Setting AFL_COMPCOV_LEVEL enables the CompareCoverage tracing of all cmp and sub in x86 and x86_64 and memory comparions functions (e.g. strcmp, memcmp, ...) when libcompcov is preloaded using AFL_PRELOAD. @@ -292,7 +292,7 @@ The QEMU wrapper used to instrument binary-only code supports several settings: only comparisons with immediate values / read-only memory and AFL_COMPCOV_LEVEL=2 that instruments all the comparions. Level 2 is more accurate but may need a larger shared memory. - + - Setting AFL_QEMU_COMPCOV enables the CompareCoverage tracing of all cmp and sub in x86 and x86_64. This is an alias of AFL_COMPCOV_LEVEL=1 when AFL_COMPCOV_LEVEL is @@ -304,25 +304,25 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - AFL_DEBUG will print the found entrypoint for the binary to stderr. Use this if you are unsure if the entrypoint might be wrong - but - use it directly, e.g. afl-qemu-trace ./program + use it directly, e.g. afl-qemu-trace ./program - AFL_ENTRYPOINT allows you to specify a specific entrypoint into the binary (this can be very good for the performance!). The entrypoint is specified as hex address, e.g. 0x4004110 Note that the address must be the address of a basic block. - + - When the target is i386/x86_64 you can specify the address of the function that has to be the body of the persistent loop using AFL_QEMU_PERSISTENT_ADDR=`start addr`. - + - Another modality to execute the persistent loop is to specify also the AFL_QEMU_PERSISTENT_RET=`end addr` env variable. With this variable assigned, instead of patching the return address, the specified instruction is transformed to a jump towards `start addr`. - + - AFL_QEMU_PERSISTENT_GPR=1 QEMU will save the original value of general purpose registers and restore them in each persistent cycle. - + - With AFL_QEMU_PERSISTENT_RETADDR_OFFSET you can specify the offset from the stack pointer in which QEMU can find the return address when `start addr` is hitted. @@ -376,7 +376,7 @@ The library honors these environmental variables: - AFL_LD_NO_CALLOC_OVER inhibits abort() on calloc() overflows. Most of the common allocators check for that internally and return NULL, so it's a security risk only in more exotic setups. - + - AFL_ALIGNED_ALLOC=1 will force the alignment of the allocation size to max_align_t to be compliant with the C standard. @@ -410,7 +410,7 @@ optimal values if not already present in the environment: - In the same vein, by default, MSAN_OPTIONS are set to: - exit_code=86 (required for legacy reasons) + exit_code=86 (required for legacy reasons) abort_on_error=1 symbolize=0 msan_track_origins=0 diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 27b22082..37b6832c 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -276,8 +276,7 @@ extern u8 cal_cycles, /* Calibration cycles defaults */ no_unlink, /* do not unlink cur_input */ use_stdin, /* use stdin for sending data */ debug, /* Debug mode */ - custom_only, /* Custom mutator only mode */ - python_only; /* Python-only mode */ + custom_only; /* Custom mutator only mode */ extern u32 stats_update_freq; /* Stats update frequency (execs) */ @@ -471,6 +470,8 @@ struct custom_mutator { * Initialize the custom mutator. * * (Optional) + * + * @param seed Seed used for the mutation. */ void (*afl_custom_init)(unsigned int seed); @@ -479,17 +480,18 @@ struct custom_mutator { * * (Optional for now. Required in the future) * - * @param[in] data Input data to be mutated - * @param[in] size Size of input data + * @param[in] buf Input data to be mutated + * @param[in] buf_size Size of input data + * @param[in] add_buf Buffer containing the additional test case + * @param[in] add_buf_size Size of the additional test case * @param[out] mutated_out Buffer to store the mutated input * @param[in] max_size Maximum size of the mutated output. The mutation must not * produce data larger than max_size. - * @param[in] seed Seed used for the mutation. The mutation should produce the - * same output given the same seed. * @return Size of the mutated output. */ - size_t (*afl_custom_fuzz)(u8* data, size_t size, u8* mutated_out, - size_t max_size, unsigned int seed); + size_t (*afl_custom_fuzz)(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size); /** * A post-processing function to use right before AFL writes the test case to @@ -498,12 +500,14 @@ struct custom_mutator { * (Optional) If this functionality is not needed, simply don't define this * function. * - * @param[in] data Buffer containing the test case to be executed - * @param[in] size Size of the test case - * @param[out] new_data Buffer to store the test case after processing - * @return Size of data after processing + * @param[in] buf Buffer containing the test case to be executed + * @param[in] buf_size Size of the test case + * @param[out] out_buf Pointer to the buffer of storing the test case after + * processing. External library should allocate memory for out_buf. AFL++ + * will release the memory after saving the test case. + * @return Size of the output buffer after processing */ - size_t (*afl_custom_pre_save)(u8* data, size_t size, u8** new_data); + size_t (*afl_custom_pre_save)(u8* buf, size_t buf_size, u8** out_buf); /** * This method is called at the start of each trimming operation and receives @@ -521,11 +525,11 @@ struct custom_mutator { * * (Optional) * - * @param data Buffer containing the test case - * @param size Size of the test case + * @param buf Buffer containing the test case + * @param buf_size Size of the test case * @return The amount of possible iteration steps to trim the input */ - u32 (*afl_custom_init_trim)(u8* data, size_t size); + u32 (*afl_custom_init_trim)(u8* buf, size_t buf_size); /** * This method is called for each trimming operation. It doesn't have any @@ -538,10 +542,12 @@ struct custom_mutator { * * (Optional) * - * @param[out] ret Buffer containing the trimmed test case - * @param[out] ret_len Size of the trimmed test case + * @param[out] out_buf Pointer to the buffer containing the trimmed test case. + * External library should allocate memory for out_buf. AFL++ will release + * the memory after saving the test case. + * @param[out] out_buf_size Pointer to the size of the trimmed test case */ - void (*afl_custom_trim)(u8** ret, size_t* ret_len); + void (*afl_custom_trim)(u8** out_buf, size_t* out_buf_size); /** * This method is called after each trim operation to inform you if your @@ -627,9 +633,9 @@ int init_py_module(u8*); void finalize_py_module(); void init_py(unsigned int seed); -/* TODO: unify fuzz interface for custom mutator and Python mutator */ -size_t fuzz_py(u8*, size_t, u8*, size_t, unsigned int); -void fuzz_py_original(char*, size_t, char*, size_t, char**, size_t*); +size_t fuzz_py(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size); size_t pre_save_py(u8* data, size_t size, u8** new_data); u32 init_trim_py(u8*, size_t); u32 post_trim_py(u8); diff --git a/include/envs.h b/include/envs.h index 306143be..791887d7 100644 --- a/include/envs.h +++ b/include/envs.h @@ -24,7 +24,7 @@ const char *afl_environment_variables[] = { "AFL_NO_X86", // not really an env but we dont want to warn on it "AFL_PATH", "AFL_PERFORMANCE_FILE", //"AFL_PERSISTENT", // not implemented anymore, so warn additionally - "AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE", "AFL_PYTHON_ONLY", + "AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE", "AFL_QEMU_COMPCOV", "AFL_QEMU_COMPCOV_DEBUG", "AFL_QEMU_DEBUG_MAPS", "AFL_QEMU_DISABLE_CACHE", "AFL_QEMU_PERSISTENT_ADDR", "AFL_QEMU_PERSISTENT_CNT", "AFL_QEMU_PERSISTENT_GPR", diff --git a/src/afl-fuzz-globals.c b/src/afl-fuzz-globals.c index a054499f..87418753 100644 --- a/src/afl-fuzz-globals.c +++ b/src/afl-fuzz-globals.c @@ -88,8 +88,7 @@ u8 cal_cycles = CAL_CYCLES, /* Calibration cycles defaults */ no_unlink, /* do not unlink cur_input */ use_stdin = 1, /* use stdin for sending data */ be_quiet, /* is AFL_QUIET set? */ - custom_only, /* Custom mutator only mode */ - python_only; /* Python-only mode */ + custom_only; /* Custom mutator only mode */ u32 stats_update_freq = 1; /* Stats update frequency (execs) */ diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 9365d487..26eaea59 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -27,6 +27,7 @@ void setup_custom_mutator(void) { + /* Try mutator library first */ u8* fn = getenv("AFL_CUSTOM_MUTATOR_LIBRARY"); if (fn) { @@ -41,6 +42,7 @@ void setup_custom_mutator(void) { return; } + /* Try Python module */ #ifdef USE_PYTHON u8* module_name = getenv("AFL_PYTHON_MODULE"); @@ -286,7 +288,7 @@ void load_custom_mutator_py(const char* module_name) { /* "afl_custom_fuzz" should not be NULL, but the interface of Python mutator is quite different from the custom mutator. */ - mutator->afl_custom_fuzz = NULL; + mutator->afl_custom_fuzz = fuzz_py; if (py_functions[PY_FUNC_PRE_SAVE]) mutator->afl_custom_pre_save = pre_save_py; diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 1e6dd45d..5d00e8df 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -482,56 +482,6 @@ u8 fuzz_one_original(char** argv) { if (use_radamsa > 1) goto radamsa_stage; - // custom_stage: // not used - yet - - if (mutator->afl_custom_fuzz) { - - stage_short = "custom"; - stage_name = "custom mutator"; - stage_max = len << 3; - stage_val_type = STAGE_VAL_NONE; - - const u32 max_seed_size = 4096 * 4096; - u8* mutated_buf = ck_alloc(max_seed_size); - - orig_hit_cnt = queued_paths + unique_crashes; - - for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) { - - size_t orig_size = (size_t)len; - size_t mutated_size = mutator->afl_custom_fuzz(in_buf, orig_size, - mutated_buf, max_seed_size, - UR(UINT32_MAX)); - if (mutated_size > 0) { - - out_buf = ck_realloc(out_buf, mutated_size); - memcpy(out_buf, mutated_buf, mutated_size); - if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) { - - goto abandon_entry; - - } - - } - - } - - ck_free(mutated_buf); - new_hit_cnt = queued_paths + unique_crashes; - - stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt; - stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max; - - if (custom_only) { - - /* Skip other stages */ - ret_val = 0; - goto abandon_entry; - - } - - } - if (cmplog_mode) { if (input_to_state_stage(argv, in_buf, out_buf, len, queue_cur->exec_cksum)) @@ -551,11 +501,7 @@ u8 fuzz_one_original(char** argv) { : havoc_max_mult * 100)) || queue_cur->passed_det) { -#ifdef USE_PYTHON - goto python_stage; -#else - goto havoc_stage; -#endif + goto custom_mutator_stage; } @@ -564,11 +510,7 @@ u8 fuzz_one_original(char** argv) { if (master_max && (queue_cur->exec_cksum % master_max) != master_id - 1) { -#ifdef USE_PYTHON - goto python_stage; -#else - goto havoc_stage; -#endif + goto custom_mutator_stage; } @@ -1583,24 +1525,25 @@ skip_extras: if (!queue_cur->passed_det) mark_as_det_done(queue_cur); -#ifdef USE_PYTHON -python_stage: - /********************************** - * EXTERNAL MUTATORS (Python API) * - **********************************/ +custom_mutator_stage: + /******************* + * CUSTOM MUTATORS * + *******************/ - if (!py_module) goto havoc_stage; + if (!mutator) goto havoc_stage; + if (!mutator->afl_custom_fuzz) goto havoc_stage; - stage_name = "python"; - stage_short = "python"; + stage_name = "custom mutator"; + stage_short = "custom"; stage_max = HAVOC_CYCLES * perf_score / havoc_div / 100; + stage_val_type = STAGE_VAL_NONE; if (stage_max < HAVOC_MIN) stage_max = HAVOC_MIN; - orig_hit_cnt = queued_paths + unique_crashes; + const u32 max_seed_size = 4096 * 4096; + u8* mutated_buf = ck_alloc(max_seed_size); - char* retbuf = NULL; - size_t retlen = 0; + orig_hit_cnt = queued_paths + unique_crashes; for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) { @@ -1647,26 +1590,24 @@ python_stage: ck_read(fd, new_buf, target->len, target->fname); close(fd); - fuzz_py_original(out_buf, len, new_buf, target->len, &retbuf, &retlen); + size_t mutated_size = mutator->afl_custom_fuzz(out_buf, len, + new_buf, target->len, + mutated_buf, max_seed_size); ck_free(new_buf); - if (retbuf) { + if (mutated_size > 0) { - if (!retlen) goto abandon_entry; + out_buf = ck_realloc(out_buf, mutated_size); + memcpy(out_buf, mutated_buf, mutated_size); - if (common_fuzz_stuff(argv, retbuf, retlen)) { + if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) { - free(retbuf); + ck_free(mutated_buf); goto abandon_entry; } - /* Reset retbuf/retlen */ - free(retbuf); - retbuf = NULL; - retlen = 0; - /* If we're finding new stuff, let's run for a bit longer, limits permitting. */ @@ -1687,12 +1628,13 @@ python_stage: } + ck_free(mutated_buf); new_hit_cnt = queued_paths + unique_crashes; - stage_finds[STAGE_PYTHON] += new_hit_cnt - orig_hit_cnt; - stage_cycles[STAGE_PYTHON] += stage_max; + stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt; + stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max; - if (python_only) { + if (custom_only) { /* Skip other stages */ ret_val = 0; @@ -1700,8 +1642,6 @@ python_stage: } -#endif - /**************** * RANDOM HAVOC * ****************/ @@ -2270,11 +2210,10 @@ retry_splicing: out_buf = ck_alloc_nozero(len); memcpy(out_buf, in_buf, len); -#ifdef USE_PYTHON - goto python_stage; -#else - goto havoc_stage; -#endif + goto custom_mutator_stage; + /* ???: While integrating Python module, the author decided to jump to + python stage, but the reason behind this is not clear.*/ + // goto havoc_stage; } diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index c8caa4c1..c22e4402 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -159,67 +159,16 @@ void init_py(unsigned int seed) { } } -void fuzz_py_original(char* buf, size_t buflen, - char* add_buf, size_t add_buflen, - char** ret, size_t* retlen) { +size_t fuzz_py(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size) { - if (py_module != NULL) { - - PyObject *py_args, *py_value; - py_args = PyTuple_New(2); - py_value = PyByteArray_FromStringAndSize(buf, buflen); - if (!py_value) { - - Py_DECREF(py_args); - fprintf(stderr, "Cannot convert argument\n"); - return; - - } - - PyTuple_SetItem(py_args, 0, py_value); - - py_value = PyByteArray_FromStringAndSize(add_buf, add_buflen); - if (!py_value) { - - Py_DECREF(py_args); - fprintf(stderr, "Cannot convert argument\n"); - return; - - } - - PyTuple_SetItem(py_args, 1, py_value); - - py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args); - - Py_DECREF(py_args); - - if (py_value != NULL) { - - *retlen = PyByteArray_Size(py_value); - *ret = malloc(*retlen); - memcpy(*ret, PyByteArray_AsString(py_value), *retlen); - Py_DECREF(py_value); - - } else { - - PyErr_Print(); - fprintf(stderr, "Call failed\n"); - return; - - } - - } - -} - -size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, - unsigned int seed) { - - size_t out_size; + size_t mutated_size; PyObject *py_args, *py_value; py_args = PyTuple_New(3); - py_value = PyByteArray_FromStringAndSize(data, size); + /* buf */ + py_value = PyByteArray_FromStringAndSize(buf, buf_size); if (!py_value) { Py_DECREF(py_args); @@ -229,11 +178,8 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, PyTuple_SetItem(py_args, 0, py_value); -#if PY_MAJOR_VERSION >= 3 - py_value = PyLong_FromLong(max_size); -#else - py_value = PyInt_FromLong(max_size); -#endif + /* add_buf */ + py_value = PyByteArray_FromStringAndSize(add_buf, add_buf_size); if (!py_value) { Py_DECREF(py_args); @@ -243,10 +189,11 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, PyTuple_SetItem(py_args, 1, py_value); + /* max_size */ #if PY_MAJOR_VERSION >= 3 - py_value = PyLong_FromLong(seed); + py_value = PyLong_FromLong(max_size); #else - py_value = PyInt_FromLong(seed); + py_value = PyInt_FromLong(max_size); #endif if (!py_value) { @@ -263,11 +210,10 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, if (py_value != NULL) { - out_size = PyByteArray_Size(py_value); - memcpy(mutated_out, PyByteArray_AsString(py_value), out_size); + mutated_size = PyByteArray_Size(py_value); + memcpy(mutated_out, PyByteArray_AsString(py_value), mutated_size); Py_DECREF(py_value); - - return out_size; + return mutated_size; } else { @@ -278,12 +224,12 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size, } -size_t pre_save_py(u8* data, size_t size, u8** new_data) { +size_t pre_save_py(u8* buf, size_t buf_size, u8** out_buf) { - size_t new_size; + size_t out_buf_size; PyObject *py_args, *py_value; py_args = PyTuple_New(2); - py_value = PyByteArray_FromStringAndSize(data, size); + py_value = PyByteArray_FromStringAndSize(buf, buf_size); if (!py_value) { Py_DECREF(py_args); @@ -299,11 +245,11 @@ size_t pre_save_py(u8* data, size_t size, u8** new_data) { if (py_value != NULL) { - new_size = PyByteArray_Size(py_value); - *new_data = malloc(new_size); - memcpy(*new_data, PyByteArray_AsString(py_value), new_size); + out_buf_size = PyByteArray_Size(py_value); + *out_buf = malloc(out_buf_size); + memcpy(*out_buf, PyByteArray_AsString(py_value), out_buf_size); Py_DECREF(py_value); - return new_size; + return out_buf_size; } else { @@ -314,12 +260,12 @@ size_t pre_save_py(u8* data, size_t size, u8** new_data) { } -u32 init_trim_py(u8* buf, size_t buflen) { +u32 init_trim_py(u8* buf, size_t buf_size) { PyObject *py_args, *py_value; py_args = PyTuple_New(1); - py_value = PyByteArray_FromStringAndSize(buf, buflen); + py_value = PyByteArray_FromStringAndSize(buf, buf_size); if (!py_value) { Py_DECREF(py_args); @@ -389,7 +335,7 @@ u32 post_trim_py(u8 success) { } -void trim_py(u8** ret, size_t* retlen) { +void trim_py(u8** out_buf, size_t* out_buf_size) { PyObject *py_args, *py_value; @@ -399,9 +345,9 @@ void trim_py(u8** ret, size_t* retlen) { if (py_value != NULL) { - *retlen = PyByteArray_Size(py_value); - *ret = malloc(*retlen); - memcpy(*ret, PyByteArray_AsString(py_value), *retlen); + *out_buf_size = PyByteArray_Size(py_value); + *out_buf = malloc(*out_buf_size); + memcpy(*out_buf, PyByteArray_AsString(py_value), *out_buf_size); Py_DECREF(py_value); } else { diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index a1d10387..12352355 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -309,11 +309,12 @@ void write_to_testcase(void* mem, u32 len) { lseek(fd, 0, SEEK_SET); - if (mutator->afl_custom_pre_save) { + if (mutator && mutator->afl_custom_pre_save) { u8* new_data; size_t new_size = mutator->afl_custom_pre_save(mem, len, &new_data); ck_write(fd, new_data, new_size, out_file); + ck_free(new_data); } else { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index d329a20e..a96ee1d0 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -158,7 +158,6 @@ static void usage(u8* argv0, int more_help) { "AFL_CUSTOM_MUTATOR_LIBRARY: lib with afl_custom_fuzz() to mutate inputs\n" "AFL_CUSTOM_MUTATOR_ONLY: avoid AFL++'s internal mutators\n" "AFL_PYTHON_MODULE: mutate and trim inputs with the specified Python module\n" - "AFL_PYTHON_ONLY: skip AFL++'s own mutators\n" "AFL_DEBUG: extra debugging output for Python mode trimming\n" "AFL_DISABLE_TRIM: disable the trimming of test cases\n" "AFL_NO_UI: switch status screen off\n" @@ -658,11 +657,10 @@ int main(int argc, char** argv, char** envp) { OKF("afl-tmin fork server patch from github.com/nccgroup/TriforceAFL"); OKF("MOpt Mutator from github.com/puppet-meteor/MOpt-AFL"); - if (sync_id && force_deterministic && - (getenv("AFL_CUSTOM_MUTATOR_ONLY") || getenv("AFL_PYTHON_ONLY"))) + if (sync_id && force_deterministic && getenv("AFL_CUSTOM_MUTATOR_ONLY")) WARNF( - "Using -M master with the AFL_..._ONLY mutator options will result in " - "no deterministic mutations being done!"); + "Using -M master with the AFL_CUSTOM_MUTATOR_ONLY mutator options will " + "result in no deterministic mutations being done!"); check_environment_vars(envp); @@ -832,16 +830,6 @@ int main(int argc, char** argv, char** envp) { if (get_afl_env("AFL_DEBUG")) debug = 1; - if (get_afl_env("AFL_PYTHON_ONLY")) { - - /* This ensures we don't proceed to havoc/splice */ - python_only = 1; - - /* Ensure we also skip all deterministic steps */ - skip_deterministic = 1; - - } - if (get_afl_env("AFL_CUSTOM_MUTATOR_ONLY")) { /* This ensures we don't proceed to havoc/splice */ -- cgit 1.4.1