diff options
| -rw-r--r-- | Makefile | 53 | ||||
| -rw-r--r-- | afl-fuzz.c | 525 | ||||
| -rw-r--r-- | docs/ChangeLog | 3 | ||||
| -rw-r--r-- | docs/env_variables.txt | 9 | ||||
| -rw-r--r-- | docs/python_mutators.txt | 147 |
5 files changed, 713 insertions, 24 deletions
diff --git a/Makefile b/Makefile index d05cd76c..1d8d819f 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,8 @@ CFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign \ -DAFL_PATH=\"$(HELPER_PATH)\" -DDOC_PATH=\"$(DOC_PATH)\" \ -DBIN_PATH=\"$(BIN_PATH)\" +PYTHON_INCLUDE ?= /usr/include/python2.7 + ifneq "$(filter Linux GNU%,$(shell uname))" "" LDFLAGS += -ldl endif @@ -44,7 +46,15 @@ endif COMM_HDR = alloc-inl.h config.h debug.h types.h -all: test_x86 $(PROGS) afl-as test_build all_done +ifeq "$(shell echo '\#include <Python.h>XXXvoid main() {}' | sed 's/XXX/\n/g' | $(CC) -x c - -o .test -I$(PYTHON_INCLUDE) -lpython2.7 && echo 1 || echo 0 )" "1" + PYTHON_OK=1 + PYFLAGS=-DUSE_PYTHON -I$(PYTHON_INCLUDE) -lpython2.7 +else + PYTHON_OK=0 + PYFLAGS= +endif + +all: test_x86 test_python27 $(PROGS) afl-as test_build all_done ifndef AFL_NO_X86 @@ -61,6 +71,19 @@ test_x86: endif +ifeq "$(PYTHON_OK)" "1" + +test_python27: + @rm -f .test 2> /dev/null + @echo "[+] Python 2.7 support seems to be working." + +else + +test_python27: + @echo "[-] You seem to need to install the package python2.7-dev, but it is optional so we continue" + +endif + afl-gcc: afl-gcc.c $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) set -e; for i in afl-g++ afl-clang afl-clang++; do ln -sf afl-gcc $$i; done @@ -70,7 +93,7 @@ afl-as: afl-as.c afl-as.h $(COMM_HDR) | test_x86 ln -sf afl-as as afl-fuzz: afl-fuzz.c $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) $(PYFLAGS) afl-showmap: afl-showmap.c $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) @@ -142,16 +165,16 @@ endif cp -r dictionaries/ $${DESTDIR}$(MISC_PATH) publish: clean - test "`basename $$PWD`" = "afl" || exit 1 - test -f ~/www/afl/releases/$(PROGNAME)-$(VERSION).tgz; if [ "$$?" = "0" ]; then echo; echo "Change program version in config.h, mmkay?"; echo; exit 1; fi - cd ..; rm -rf $(PROGNAME)-$(VERSION); cp -pr $(PROGNAME) $(PROGNAME)-$(VERSION); \ - tar -cvz -f ~/www/afl/releases/$(PROGNAME)-$(VERSION).tgz $(PROGNAME)-$(VERSION) - chmod 644 ~/www/afl/releases/$(PROGNAME)-$(VERSION).tgz - ( cd ~/www/afl/releases/; ln -s -f $(PROGNAME)-$(VERSION).tgz $(PROGNAME)-latest.tgz ) - cat docs/README >~/www/afl/README.txt - cat docs/status_screen.txt >~/www/afl/status_screen.txt - cat docs/historical_notes.txt >~/www/afl/historical_notes.txt - cat docs/technical_details.txt >~/www/afl/technical_details.txt - cat docs/ChangeLog >~/www/afl/ChangeLog.txt - cat docs/QuickStartGuide.txt >~/www/afl/QuickStartGuide.txt - echo -n "$(VERSION)" >~/www/afl/version.txt +# test "`basename $$PWD`" = "afl" || exit 1 +# test -f ~/www/afl/releases/$(PROGNAME)-$(VERSION).tgz; if [ "$$?" = "0" ]; then echo; echo "Change program version in config.h, mmkay?"; echo; exit 1; fi +# cd ..; rm -rf $(PROGNAME)-$(VERSION); cp -pr $(PROGNAME) $(PROGNAME)-$(VERSION); \ +# tar -cvz -f ~/www/afl/releases/$(PROGNAME)-$(VERSION).tgz $(PROGNAME)-$(VERSION) +# chmod 644 ~/www/afl/releases/$(PROGNAME)-$(VERSION).tgz +# ( cd ~/www/afl/releases/; ln -s -f $(PROGNAME)-$(VERSION).tgz $(PROGNAME)-latest.tgz ) +# cat docs/README >~/www/afl/README.txt +# cat docs/status_screen.txt >~/www/afl/status_screen.txt +# cat docs/historical_notes.txt >~/www/afl/historical_notes.txt +# cat docs/technical_details.txt >~/www/afl/technical_details.txt +# cat docs/ChangeLog >~/www/afl/ChangeLog.txt +# cat docs/QuickStartGuide.txt >~/www/afl/QuickStartGuide.txt +# echo -n "$(VERSION)" >~/www/afl/version.txt diff --git a/afl-fuzz.c b/afl-fuzz.c index b775803d..ef95280a 100644 --- a/afl-fuzz.c +++ b/afl-fuzz.c @@ -98,6 +98,11 @@ static u32 hang_tmout = EXEC_TIMEOUT; /* Timeout used for hang det (ms) */ EXP_ST u64 mem_limit = MEM_LIMIT; /* Memory cap for child (MB) */ +EXP_ST u8 cal_cycles = CAL_CYCLES; /* Calibration cycles defaults */ +EXP_ST u8 cal_cycles_long = CAL_CYCLES_LONG; +EXP_ST u8 debug, /* Debug mode */ + python_only; /* Python-only mode */ + static u32 stats_update_freq = 1; /* Stats update frequency (execs) */ enum { @@ -313,7 +318,8 @@ enum { /* 13 */ STAGE_EXTRAS_UI, /* 14 */ STAGE_EXTRAS_AO, /* 15 */ STAGE_HAVOC, - /* 16 */ STAGE_SPLICE + /* 16 */ STAGE_SPLICE, + /* 17 */ STAGE_PYTHON }; /* Stage value types */ @@ -336,6 +342,222 @@ enum { }; +static inline u32 UR(u32 limit); + +/* Python stuff */ +#ifdef USE_PYTHON +#include <Python.h> + +static PyObject *py_module; + +enum { + /* 00 */ PY_FUNC_INIT, + /* 01 */ PY_FUNC_FUZZ, + /* 02 */ PY_FUNC_INIT_TRIM, + /* 03 */ PY_FUNC_POST_TRIM, + /* 04 */ PY_FUNC_TRIM, + PY_FUNC_COUNT +}; + +static PyObject *py_functions[PY_FUNC_COUNT]; + +static int init_py() { + Py_Initialize(); + u8* module_name = getenv("AFL_PYTHON_MODULE"); + u8 py_notrim = 0; + + if (module_name) { + PyObject* py_name = PyString_FromString(module_name); + + py_module = PyImport_Import(py_name); + Py_DECREF(py_name); + + if (py_module != NULL) { + py_functions[PY_FUNC_INIT] = PyObject_GetAttrString(py_module, "init"); + py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "fuzz"); + py_functions[PY_FUNC_INIT_TRIM] = PyObject_GetAttrString(py_module, "init_trim"); + py_functions[PY_FUNC_POST_TRIM] = PyObject_GetAttrString(py_module, "post_trim"); + py_functions[PY_FUNC_TRIM] = PyObject_GetAttrString(py_module, "trim"); + + for (u8 py_idx = 0; py_idx < PY_FUNC_COUNT; ++py_idx) { + if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { + if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { + // Implementing the trim API is optional for now + if (PyErr_Occurred()) + PyErr_Print(); + py_notrim = 1; + } else { + if (PyErr_Occurred()) + PyErr_Print(); + fprintf(stderr, "Cannot find/call function with index %d in external Python module.\n", py_idx); + return 1; + } + } + + } + + if (py_notrim) { + py_functions[PY_FUNC_INIT_TRIM] = NULL; + py_functions[PY_FUNC_POST_TRIM] = NULL; + py_functions[PY_FUNC_TRIM] = NULL; + WARNF("Python module does not implement trim API, standard trimming will be used."); + } + + PyObject *py_args, *py_value; + + /* Provide the init function a seed for the Python RNG */ + py_args = PyTuple_New(1); + py_value = PyInt_FromLong(UR(0xFFFFFFFF)); + if (!py_value) { + Py_DECREF(py_args); + fprintf(stderr, "Cannot convert argument\n"); + return 1; + } + + PyTuple_SetItem(py_args, 0, py_value); + + py_value = PyObject_CallObject(py_functions[PY_FUNC_INIT], py_args); + + Py_DECREF(py_args); + + if (py_value == NULL) { + PyErr_Print(); + fprintf(stderr,"Call failed\n"); + return 1; + } + } else { + PyErr_Print(); + fprintf(stderr, "Failed to load \"%s\"\n", module_name); + return 1; + } + } + + return 0; +} + +static void finalize_py() { + if (py_module != NULL) { + u32 i; + for (i = 0; i < PY_FUNC_COUNT; ++i) + Py_XDECREF(py_functions[i]); + + Py_DECREF(py_module); + } + + Py_Finalize(); +} + +static void fuzz_py(char* buf, size_t buflen, char* add_buf, size_t add_buflen, char** ret, size_t* retlen) { + PyObject *py_args, *py_value; + + if (py_module != NULL) { + py_args = PyTuple_New(2); + py_value = PyByteArray_FromStringAndSize(buf, buflen); + if (!py_value) { + Py_DECREF(py_args); + fprintf(stderr, "Cannot convert argument\n"); + return; + } + + PyTuple_SetItem(py_args, 0, py_value); + + py_value = PyByteArray_FromStringAndSize(add_buf, add_buflen); + if (!py_value) { + Py_DECREF(py_args); + fprintf(stderr, "Cannot convert argument\n"); + return; + } + + PyTuple_SetItem(py_args, 1, py_value); + + py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args); + + Py_DECREF(py_args); + + if (py_value != NULL) { + *retlen = PyByteArray_Size(py_value); + *ret = malloc(*retlen); + memcpy(*ret, PyByteArray_AsString(py_value), *retlen); + Py_DECREF(py_value); + } else { + PyErr_Print(); + fprintf(stderr,"Call failed\n"); + return; + } + } +} +static u32 init_trim_py(char* buf, size_t buflen) { + PyObject *py_args, *py_value; + + py_args = PyTuple_New(1); + py_value = PyByteArray_FromStringAndSize(buf, buflen); + if (!py_value) { + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + } + + PyTuple_SetItem(py_args, 0, py_value); + + py_value = PyObject_CallObject(py_functions[PY_FUNC_INIT_TRIM], py_args); + Py_DECREF(py_args); + + if (py_value != NULL) { + u32 retcnt = PyInt_AsLong(py_value); + Py_DECREF(py_value); + return retcnt; + } else { + PyErr_Print(); + FATAL("Call failed"); + } +} +static u32 post_trim_py(char success) { + PyObject *py_args, *py_value; + + py_args = PyTuple_New(1); + + py_value = PyBool_FromLong(success); + if (!py_value) { + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); + } + + PyTuple_SetItem(py_args, 0, py_value); + + py_value = PyObject_CallObject(py_functions[PY_FUNC_POST_TRIM], py_args); + Py_DECREF(py_args); + + if (py_value != NULL) { + u32 retcnt = PyInt_AsLong(py_value); + Py_DECREF(py_value); + return retcnt; + } else { + PyErr_Print(); + FATAL("Call failed"); + } +} + +static void trim_py(char** ret, size_t* retlen) { + PyObject *py_args, *py_value; + + py_args = PyTuple_New(0); + py_value = PyObject_CallObject(py_functions[PY_FUNC_TRIM], py_args); + Py_DECREF(py_args); + + if (py_value != NULL) { + *retlen = PyByteArray_Size(py_value); + *ret = malloc(*retlen); + memcpy(*ret, PyByteArray_AsString(py_value), *retlen); + Py_DECREF(py_value); + } else { + PyErr_Print(); + FATAL("Call failed"); + } +} + +#endif /* USE_PYTHON */ + + + /* Get unix time in milliseconds */ static u64 get_cur_time(void) { @@ -2071,8 +2293,10 @@ EXP_ST void init_forkserver(char** argv) { setsid(); - dup2(dev_null_fd, 1); - dup2(dev_null_fd, 2); + if (!getenv("AFL_DEBUG_CHILD_OUTPUT")) { + dup2(dev_null_fd, 1); + dup2(dev_null_fd, 2); + } if (out_file) { @@ -3892,6 +4116,10 @@ static void maybe_delete_out_dir(void) { fn = alloc_printf("%s/plot_data", out_dir); if (unlink(fn) && errno != ENOENT) goto dir_cleanup_failed; ck_free(fn); + + fn = alloc_printf("%s/cmdline", out_dir); + if (unlink(fn) && errno != ENOENT) goto dir_cleanup_failed; + ck_free(fn); OKF("Output dir cleanup successful."); @@ -4294,9 +4522,10 @@ static void show_stats(void) { " imported : " cRST "%-10s" bSTG bV "\n", tmp, sync_id ? DI(queued_imported) : (u8*)"n/a"); - sprintf(tmp, "%s/%s, %s/%s", + sprintf(tmp, "%s/%s, %s/%s, %s/%s", DI(stage_finds[STAGE_HAVOC]), DI(stage_cycles[STAGE_HAVOC]), - DI(stage_finds[STAGE_SPLICE]), DI(stage_cycles[STAGE_SPLICE])); + DI(stage_finds[STAGE_SPLICE]), DI(stage_cycles[STAGE_SPLICE]), + DI(stage_finds[STAGE_PYTHON]), DI(stage_cycles[STAGE_PYTHON])); SAYF(bV bSTOP " havoc : " cRST "%-36s " bSTG bV bSTOP, tmp); @@ -4497,6 +4726,113 @@ static void show_init_stats(void) { } +#ifdef USE_PYTHON +static u8 trim_case_python(char** argv, struct queue_entry* q, u8* in_buf) { + + static u8 tmp[64]; + static u8 clean_trace[MAP_SIZE]; + + u8 needs_write = 0, fault = 0; + u32 trim_exec = 0; + u32 orig_len = q->len; + + stage_name = tmp; + bytes_trim_in += q->len; + + /* Initialize trimming in the Python module */ + stage_cur = 0; + stage_max = init_trim_py(in_buf, q->len); + + if (not_on_tty && debug) + SAYF("[Python Trimming] START: Max %d iterations, %d bytes", stage_max, q->len); + + while(stage_cur < stage_max) { + sprintf(tmp, "ptrim %s", DI(trim_exec)); + + u32 cksum; + + char* retbuf = NULL; + size_t retlen = 0; + + trim_py(&retbuf, &retlen); + + if (retlen > orig_len) + FATAL("Trimmed data returned by Python module is larger than original data"); + + write_to_testcase(retbuf, retlen); + + fault = run_target(argv, exec_tmout); + trim_execs++; + + if (stop_soon || fault == FAULT_ERROR) goto abort_trimming; + + cksum = hash32(trace_bits, MAP_SIZE, HASH_CONST); + + if (cksum == q->exec_cksum) { + + q->len = retlen; + memcpy(in_buf, retbuf, retlen); + + /* Let's save a clean trace, which will be needed by + update_bitmap_score once we're done with the trimming stuff. */ + + if (!needs_write) { + + needs_write = 1; + memcpy(clean_trace, trace_bits, MAP_SIZE); + + } + + /* Tell the Python module that the trimming was successful */ + stage_cur = post_trim_py(1); + + if (not_on_tty && debug) + SAYF("[Python Trimming] SUCCESS: %d/%d iterations (now at %d bytes)", stage_cur, stage_max, q->len); + } else { + /* Tell the Python module that the trimming was unsuccessful */ + stage_cur = post_trim_py(0); + if (not_on_tty && debug) + SAYF("[Python Trimming] FAILURE: %d/%d iterations", stage_cur, stage_max); + } + + /* Since this can be slow, update the screen every now and then. */ + + if (!(trim_exec++ % stats_update_freq)) show_stats(); + } + + if (not_on_tty && debug) + SAYF("[Python Trimming] DONE: %d bytes -> %d bytes", orig_len, q->len); + + /* If we have made changes to in_buf, we also need to update the on-disk + version of the test case. */ + + if (needs_write) { + + s32 fd; + + unlink(q->fname); /* ignore errors */ + + fd = open(q->fname, O_WRONLY | O_CREAT | O_EXCL, 0600); + + if (fd < 0) PFATAL("Unable to create '%s'", q->fname); + + ck_write(fd, in_buf, q->len, q->fname); + close(fd); + + memcpy(trace_bits, clean_trace, MAP_SIZE); + update_bitmap_score(q); + + } + + + +abort_trimming: + + bytes_trim_out += q->len; + return fault; + +} +#endif /* Trim all new test cases to save cycles when doing deterministic checks. The trimmer uses power-of-two increments somewhere between 1/16 and 1/1024 of @@ -4504,6 +4840,11 @@ static void show_init_stats(void) { static u8 trim_case(char** argv, struct queue_entry* q, u8* in_buf) { +#ifdef USE_PYTHON + if (py_functions[PY_FUNC_TRIM]) + return trim_case_python(argv, q, in_buf); +#endif + static u8 tmp[64]; static u8 clean_trace[MAP_SIZE]; @@ -5044,7 +5385,7 @@ static u8 fuzz_one(char** argv) { s32 len, fd, temp_len, i, j; u8 *in_buf, *out_buf, *orig_in, *ex_tmp, *eff_map = 0; - u64 havoc_queued, orig_hit_cnt, new_hit_cnt; + u64 havoc_queued = 0, orig_hit_cnt, new_hit_cnt; u32 splice_cycle = 0, perf_score = 100, orig_perf, prev_cksum, eff_cnt = 1; u8 ret_val = 1, doing_det = 0; @@ -5106,7 +5447,7 @@ static u8 fuzz_one(char** argv) { orig_in = in_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - if (orig_in == MAP_FAILED) PFATAL("Unable to mmap '%s'", queue_cur->fname); + if (orig_in == MAP_FAILED) PFATAL("Unable to mmap '%s' with len %u", queue_cur->fname, len); close(fd); @@ -5190,13 +5531,21 @@ static u8 fuzz_one(char** argv) { ? queue_cur->depth * 30 : HAVOC_MAX_MULT * 100)) || queue_cur->passed_det) +#ifdef USE_PYTHON + goto python_stage; +#else goto havoc_stage; +#endif /* Skip deterministic fuzzing if exec path checksum puts this out of scope for this master instance. */ if (master_max && (queue_cur->exec_cksum % master_max) != master_id - 1) +#ifdef USE_PYTHON + goto python_stage; +#else goto havoc_stage; +#endif doing_det = 1; @@ -6155,6 +6504,99 @@ skip_extras: if (!queue_cur->passed_det) mark_as_det_done(queue_cur); +#ifdef USE_PYTHON +python_stage: + /********************************** + * EXTERNAL MUTATORS (Python API) * + **********************************/ + + if (!py_module) goto havoc_stage; + + stage_name = "python"; + stage_short = "python"; + stage_max = HAVOC_CYCLES * perf_score / havoc_div / 100; + + if (stage_max < HAVOC_MIN) stage_max = HAVOC_MIN; + + orig_hit_cnt = queued_paths + unique_crashes; + + char* retbuf = NULL; + size_t retlen = 0; + + for (stage_cur = 0; stage_cur < stage_max; stage_cur++) { + struct queue_entry* target; + u32 tid; + u8* new_buf; + +retry_external_pick: + /* Pick a random other queue entry for passing to external API */ + do { tid = UR(queued_paths); } while (tid == current_entry && queued_paths > 1); + + target = queue; + + while (tid >= 100) { target = target->next_100; tid -= 100; } + while (tid--) target = target->next; + + /* Make sure that the target has a reasonable length. */ + + while (target && (target->len < 2 || target == queue_cur) && queued_paths > 1) { + target = target->next; + splicing_with++; + } + + if (!target) goto retry_external_pick; + + /* Read the additional testcase into a new buffer. */ + fd = open(target->fname, O_RDONLY); + if (fd < 0) PFATAL("Unable to open '%s'", target->fname); + new_buf = ck_alloc_nozero(target->len); + ck_read(fd, new_buf, target->len, target->fname); + close(fd); + + fuzz_py(out_buf, len, new_buf, target->len, &retbuf, &retlen); + + ck_free(new_buf); + + if (retbuf) { + if (!retlen) + goto abandon_entry; + + if (common_fuzz_stuff(argv, retbuf, retlen)) { + free(retbuf); + goto abandon_entry; + } + + /* Reset retbuf/retlen */ + free(retbuf); + retbuf = NULL; + retlen = 0; + + /* If we're finding new stuff, let's run for a bit longer, limits + permitting. */ + + if (queued_paths != havoc_queued) { + if (perf_score <= HAVOC_MAX_MULT * 100) { + stage_max *= 2; + perf_score *= 2; + } + + havoc_queued = queued_paths; + } + } + } + + new_hit_cnt = queued_paths + unique_crashes; + + stage_finds[STAGE_PYTHON] += new_hit_cnt - orig_hit_cnt; + stage_cycles[STAGE_PYTHON] += stage_max; + + if (python_only) { + /* Skip other stages */ + ret_val = 0; + goto abandon_entry; + } +#endif + /**************** * RANDOM HAVOC * ****************/ @@ -6699,7 +7141,11 @@ retry_splicing: out_buf = ck_alloc_nozero(len); memcpy(out_buf, in_buf, len); +#ifdef USE_PYTHON + goto python_stage; +#else goto havoc_stage; +#endif } @@ -7180,6 +7626,9 @@ static void usage(u8* argv0) { " -C - crash exploration mode (the peruvian rabbit thing)\n" " -e ext - File extension for the temporarily generated test case\n\n" +#ifdef USE_PYTHON + "Compiled with Python 2.7 module support, see docs/python_mutators.txt\n" +#endif "For additional tips, please consult %s/README.\n\n", argv0, EXEC_TIMEOUT, MEM_LIMIT, doc_path); @@ -7311,6 +7760,30 @@ EXP_ST void setup_dirs_fds(void) { } +static void setup_cmdline_file(char** argv) { + u8* tmp; + s32 fd; + u32 i = 0; + + FILE* cmdline_file = NULL; + + /* Store the command line to reproduce our findings */ + tmp = alloc_printf("%s/cmdline", out_dir); + fd = open(tmp, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (fd < 0) PFATAL("Unable to create '%s'", tmp); + ck_free(tmp); + + cmdline_file = fdopen(fd, "w"); + if (!cmdline_file) PFATAL("fdopen() failed"); + + while (argv[i]) { + fprintf(cmdline_file, "%s\n", argv[i]); + i++; + } + + fclose(cmdline_file); +} + /* Setup the output file for fuzzed data, if not using -f. */ @@ -7557,8 +8030,8 @@ static void fix_up_sync(void) { if (force_deterministic) FATAL("use -S instead of -M -d"); - else - FATAL("-S already implies -d"); + //else + // FATAL("-S already implies -d"); } @@ -8116,6 +8589,23 @@ int main(int argc, char** argv) { check_if_tty(); + if (getenv("AFL_CAL_FAST")) { + /* Use less calibration cycles, for slow applications */ + cal_cycles = 3; + cal_cycles_long = 5; + } + + if (getenv("AFL_DEBUG")) + debug = 1; + + if (getenv("AFL_PYTHON_ONLY")) { + /* This ensures we don't proceed to havoc/splice */ + python_only = 1; + + /* Ensure we also skip all deterministic steps */ + skip_deterministic = 1; + } + get_core_count(); #ifdef HAVE_AFFINITY @@ -8130,6 +8620,19 @@ int main(int argc, char** argv) { init_count_class16(); setup_dirs_fds(); + + u8 with_python_support = 0; +#ifdef USE_PYTHON + if (init_py()) + FATAL("Failed to initialize Python module"); + with_python_support = 1; +#endif + + if (getenv("AFL_PYTHON_MODULE") && !with_python_support) + FATAL("Your AFL binary was built without Python support"); + + setup_cmdline_file(argv + optind); + read_testcases(); load_auto(); @@ -8262,6 +8765,10 @@ stop_fuzzing: alloc_report(); +#ifdef USE_PYTHON + finalize_py(); +#endif + OKF("We're done here. Have a nice day!\n"); exit(0); diff --git a/docs/ChangeLog b/docs/ChangeLog index 3d93821f..616dd78b 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -17,6 +17,9 @@ sending a mail to <afl-users+subscribe@googlegroups.com>. Version ++2.52d (tbd): ----------------------------- + - added Python Module mutator support, python2.7-dev is autodetected. + see docs/python_mutators.txt (originally by choller@mozilla) + - added AFL_CAL_FAST for slow applications and AFL_DEBUG_CHILD_OUTPUT for debugging - ... your idea or patch? diff --git a/docs/env_variables.txt b/docs/env_variables.txt index 8c9bfa95..07a74dc0 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -68,6 +68,15 @@ tools make fairly broad use of environmental variables: - Setting AFL_QUIET will prevent afl-cc and afl-as banners from being displayed during compilation, in case you find them distracting. + - Setting AFL_CAL_FAST will speed up the initial calibration, if the + application is very slow + + - Setting AFL_DEBUG_CHILD_OUTPUT will not suppress the child output. + Not pretty but good for debugging purposes. + + - For AFL_PYTHON_MODULE and AFL_PYTHON_ONLY - they require to be compiled + with -DUSE_PYTHON. Please see docs/python_mutators.txt + 2) Settings for afl-clang-fast ------------------------------ diff --git a/docs/python_mutators.txt b/docs/python_mutators.txt new file mode 100644 index 00000000..04fd7208 --- /dev/null +++ b/docs/python_mutators.txt @@ -0,0 +1,147 @@ +================================================== +Adding custom mutators to AFL using Python modules +================================================== + + This file describes how you can utilize the external Python API to write + your own custom mutation routines. + + Note: This feature is highly experimental. Use at your own risk. + + Implemented by Christian Holler (:decoder) <choller@mozilla.com>. + + NOTE: This is for Python 2.7 ! + Anyone who wants to add Python 3.7 support is happily welcome :) + + +1) Description and purpose +-------------------------- + +While AFLFuzz comes with a good selection of generic deterministic and +non-deterministic mutation operations, it sometimes might make sense to extend +these to implement strategies more specific to the target you are fuzzing. + +For simplicity and in order to allow people without C knowledge to extend +AFLFuzz, I implemented a "Python" stage that can make use of an external +module (written in Python) that implements a custom mutation stage. + +The main motivation behind this is to lower the barrier for people +experimenting with this tool. Hopefully, someone will be able to do useful +things with this extension. + +If you find it useful, have questions or need additional features added to the +interface, feel free to send a mail to <choller@mozilla.com>. + +See the following information to get a better pictures: + https://www.agarri.fr/docs/XML_Fuzzing-NullCon2017-PUBLIC.pdf + https://bugs.chromium.org/p/chromium/issues/detail?id=930663 + + +2) How the Python module looks like +----------------------------------- + +You can find a simple example in pymodules/example.py including documentation +explaining each function. In the same directory, you can find another simple +module that performs simple mutations. + +Right now, "init" is called at program startup and can be used to perform any +kinds of one-time initializations while "fuzz" is called each time a mutation +is requested. + +There is also optional support for a trimming API, see the section below for +further information about this feature. + +3) How to compile AFLFuzz with Python support +--------------------------------------------- + +You must install the python 2.7 development package of your Linux distribution +before this will work. On Debian/Ubuntu/Kali this can be done with: + apt install python2.7-dev + +A prerequisite for using this mode is to compile AFLFuzz with Python support. + +The afl Makefile performs some magic and detects Python 2.7 if it is in the +default path and compiles afl-fuzz with the feature if available (which is +/usr/include/python2.7 for the Python.h include and /usr/lib/x86_64-linux-gnu +for the libpython2.7.a library) + +In case your setup is different set the necessary variables like this: +PYTHON_INCLUDE=/path/to/python2.7/include LDFLAGS=-L/path/to/python2.7/lib make + + +4) How to run AFLFuzz with your custom module +--------------------------------------------- + +You must pass the module name inside the env variable AFL_PYTHON_MODULE. + +In addition, if you are trying to load the module from the local directory, +you must adjust your PYTHONPATH to reflect this circumstance. The following +command should work if you are inside the aflfuzz directory: + +$ AFL_PYTHON_MODULE="pymodules.test" PYTHONPATH=. ./afl-fuzz + +Optionally, the following environment variables are supported: + +AFL_PYTHON_ONLY - Disable all other mutation stages. This can prevent broken + testcases (those that your Python module can't work with + anymore) to fill up your queue. Best combined with a custom + trimming routine (see below) because trimming can cause the + same test breakage like havoc and splice. + +AFL_DEBUG - When combined with AFL_NO_UI, this causes the C trimming code + to emit additional messages about the performance and actions + of your custom Python trimmer. Use this to see if it works :) + +5) Order and statistics +----------------------- + +The Python stage is set to be the first non-deterministic stage (right before +the havoc stage). In the statistics however, it shows up as the third number +under "havoc". That's because I'm lazy and I didn't want to mess with the UI +too much ;) + +6) Trimming support +------------------- + +The generic trimming routines implemented in AFLFuzz can easily destroy the +structure of complex formats, possibly leading to a point where you have a lot +of testcases in the queue that your Python module cannot process anymore but +your target application still accepts. This is especially the case when your +target can process a part of the input (causing coverage) and then errors out +on the remaining input. + +In such cases, it makes sense to implement a custom trimming routine in Python. +The API consists of multiple methods because after each trimming step, we have +to go back into the C code to check if the coverage bitmap is still the same +for the trimmed input. Here's a quick API description: + +init_trim: This method is called at the start of each trimming operation + and receives the initial buffer. It should return the amount + of iteration steps possible on this input (e.g. if your input + has n elements and you want to remove them one by one, return n, + if you do a binary search, return log(n), and so on...). + + If your trimming algorithm doesn't allow you to determine the + amount of (remaining) steps easily (esp. while running), then you + can alternatively return 1 here and always return 0 in post_trim + until you are finished and no steps remain. In that case, + returning 1 in post_trim will end the trimming routine. The whole + current index/max iterations stuff is only used to show progress. + +trim: This method is called for each trimming operation. It doesn't + have any arguments because we already have the initial buffer + from init_trim and we can memorize the current state in global + variables. This can also save reparsing steps for each iteration. + It should return the trimmed input buffer, where the returned data + must not exceed the initial input data in length. Returning anything + that is larger than the original data (passed to init_trim) will + result in a fatal abort of AFLFuzz. + +post_trim: This method is called after each trim operation to inform you + if your trimming step was successful or not (in terms of coverage). + If you receive a failure here, you should reset your input to the + last known good state. + In any case, this method must return the next trim iteration index + (from 0 to the maximum amount of steps you returned in init_trim). + +Omitting any of the methods will cause Python trimming to be disabled and +trigger a fallback to the builtin default trimming routine. |