From 6ed0a2b4aa0c22f95ba723cfda5b1fb79a224bc8 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 10 Jun 2024 18:22:06 +0200 Subject: fast resume setup detection --- src/afl-fuzz-stats.c | 8 +++- src/afl-fuzz.c | 100 +++++++++++++++++++++++++++++++++++++++----------- src/afl-performance.c | 2 + 3 files changed, 88 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 3a71e158..a20c46d0 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -76,7 +76,13 @@ char *get_fuzzing_state(afl_state_t *afl) { void write_setup_file(afl_state_t *afl, u32 argc, char **argv) { - u8 fn[PATH_MAX]; + u8 fn[PATH_MAX], fn2[PATH_MAX]; + + snprintf(fn2, PATH_MAX, "%s/target_hash", afl->out_dir); + FILE *f2 = create_ffile(fn2); + fprintf(f2, "%p\n", (void *)get_binary_hash(afl->fsrv.target_path)); + fclose(f2); + snprintf(fn, PATH_MAX, "%s/fuzzer_setup", afl->out_dir); FILE *f = create_ffile(fn); u32 i; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index a7ddef6e..bb05c9c6 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2101,45 +2101,105 @@ int main(int argc, char **argv_orig, char **envp) { } - write_setup_file(afl, argc, argv); - setup_cmdline_file(afl, argv + optind); + check_binary(afl, argv[optind]); - read_testcases(afl, NULL); - // read_foreign_testcases(afl, 1); for the moment dont do this - OKF("Loaded a total of %u seeds.", afl->queued_items); + u64 prev_target_hash = 0; + s32 fast_resume = 0, fr_fd = -1; + + if (afl->in_place_resume) { - pivot_inputs(afl); + u8 fn[PATH_MAX], buf[32]; + snprintf(fn, PATH_MAX, "%s/target_hash", afl->out_dir); + fr_fd = open(fn, O_RDONLY); + if (fr_fd >= 0) { - if (!afl->timeout_given) { find_timeout(afl); } // only for resumes! + if (read(fr_fd, buf, 32) >= 16) { - if (afl->afl_env.afl_tmpdir && !afl->in_place_resume) { + sscanf(buf, "%p", (void**)&prev_target_hash); - char tmpfile[PATH_MAX]; + } - if (afl->file_extension) { + close(fr_fd); - snprintf(tmpfile, PATH_MAX, "%s/.cur_input.%s", afl->tmp_dir, - afl->file_extension); + } + + } + + + write_setup_file(afl, argc, argv); + + if (afl->in_place_resume) { + + u64 target_hash = get_binary_hash(afl->fsrv.target_path); + + if (!target_hash || prev_target_hash != target_hash) { + + ACTF("Target binary is different, cannot perform FAST RESUME!"); } else { - snprintf(tmpfile, PATH_MAX, "%s/.cur_input", afl->tmp_dir); + u8 fn[PATH_MAX]; + snprintf(fn, PATH_MAX, "%s/fastresume.bin", afl->out_dir); + if ((fr_fd = open(fn, O_RDONLY)) >= 0) { + + OKF("Performing FAST RESUME"); + // fast_resume = 1; + + } else { + + ACTF("fastresume.bin not found, cannot perform FAST RESUME!"); + + } } - /* there is still a race condition here, but well ... */ - if (access(tmpfile, F_OK) != -1) { + } - FATAL( - "AFL_TMPDIR already has an existing temporary input file: %s - if " - "this is not from another instance, then just remove the file.", - tmpfile); + if (fast_resume) { + + // XXX + + } else { + + read_testcases(afl, NULL); + + pivot_inputs(afl); + + if (!afl->timeout_given) { find_timeout(afl); } // only for resumes! + + if (afl->afl_env.afl_tmpdir && !afl->in_place_resume) { + + char tmpfile[PATH_MAX]; + + if (afl->file_extension) { + + snprintf(tmpfile, PATH_MAX, "%s/.cur_input.%s", afl->tmp_dir, + afl->file_extension); + + } else { + + snprintf(tmpfile, PATH_MAX, "%s/.cur_input", afl->tmp_dir); + + } + + /* there is still a race condition here, but well ... */ + if (access(tmpfile, F_OK) != -1) { + + FATAL( + "AFL_TMPDIR already has an existing temporary input file: %s - if " + "this is not from another instance, then just remove the file.", + tmpfile); + + } } } + // read_foreign_testcases(afl, 1); for the moment dont do this + OKF("Loaded a total of %u seeds.", afl->queued_items); + /* If we don't have a file name chosen yet, use a safe default. */ if (!afl->fsrv.out_file) { @@ -2196,8 +2256,6 @@ int main(int argc, char **argv_orig, char **envp) { } - check_binary(afl, argv[optind]); - #ifdef AFL_PERSISTENT_RECORD if (unlikely(afl->fsrv.persistent_record)) { diff --git a/src/afl-performance.c b/src/afl-performance.c index e8ece6b5..b824fd35 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -99,11 +99,13 @@ inline u64 hash64(u8 *key, u32 len, u64 seed) { u64 get_binary_hash(u8 *fn) { + if (!fn) { return 0; } int fd = open(fn, O_RDONLY); if (fd < 0) { PFATAL("Unable to open '%s'", fn); } struct stat st; if (fstat(fd, &st) < 0) { PFATAL("Unable to fstat '%s'", fn); } u32 f_len = st.st_size; + if (!f_len) { return 0; } u8 *f_data = mmap(0, f_len, PROT_READ, MAP_PRIVATE, fd, 0); if (f_data == MAP_FAILED) { PFATAL("Unable to mmap file '%s'", fn); } close(fd); -- cgit 1.4.1 From dd762726dc7055f4b1c48da2ee1b22ff6fdde35e Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 12 Jun 2024 09:10:35 +0200 Subject: fastresume implementation --- include/afl-fuzz.h | 13 ++- src/afl-fuzz-init.c | 2 +- src/afl-fuzz-queue.c | 7 +- src/afl-fuzz.c | 224 +++++++++++++++++++++++++++++++++++++++++---------- 4 files changed, 189 insertions(+), 57 deletions(-) (limited to 'src') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index e3e4e246..3f3d6da0 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -236,7 +236,6 @@ struct queue_entry { custom, /* Marker for custom mutators */ stats_mutated; /* stats: # of mutations performed */ - u8 *trace_mini; /* Trace bytes, if kept */ u32 tc_ref; /* Trace bytes ref count */ #ifdef INTROSPECTION @@ -246,13 +245,11 @@ struct queue_entry { double perf_score, /* performance score */ weight; - u8 *testcase_buf; /* The testcase buffer, if loaded. */ - - u8 *cmplog_colorinput; /* the result buf of colorization */ - struct tainted *taint; /* Taint information from CmpLog */ - - struct queue_entry *mother; /* queue entry this based on */ - + struct queue_entry *mother; /* queue entry this based on */ + u8 *trace_mini; /* Trace bytes, if kept */ + u8 *testcase_buf; /* The testcase buffer, if loaded. */ + u8 *cmplog_colorinput; /* the result buf of colorization */ + struct tainted *taint; /* Taint information from CmpLog */ struct skipdet_entry *skipdet_e; }; diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 7310e49f..98de26dd 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -1019,7 +1019,7 @@ void perform_dry_run(afl_state_t *afl) { } - if (!q->was_fuzzed) { + if (unlikely(!q->was_fuzzed)) { q->was_fuzzed = 1; afl->reinit_table = 1; diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index f4cb930d..a28172f9 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -401,7 +401,7 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) { } else { - if (unlink(fn)) { PFATAL("Unable to remove '%s'", fn); } + if (unlink(fn)) { /*PFATAL("Unable to remove '%s'", fn);*/ } } @@ -699,12 +699,11 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { void destroy_queue(afl_state_t *afl) { - u32 i; + u32 i; + struct queue_entry *q; for (i = 0; i < afl->queued_items; i++) { - struct queue_entry *q; - q = afl->queue_buf[i]; ck_free(q->fname); ck_free(q->trace_mini); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index bb05c9c6..d8be5721 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -181,7 +181,7 @@ static void usage(u8 *argv0, int more_help) { "it.\n" " if using QEMU/FRIDA or the fuzzing target is " "compiled\n" - " for CmpLog then use '-c 0'. To disable Cmplog use '-c " + " for CmpLog then use '-c 0'. To disable CMPLOG use '-c " "-'.\n" " -l cmplog_opts - CmpLog configuration values (e.g. \"2ATR\"):\n" " 1=small files, 2=larger files (default), 3=all " @@ -2116,7 +2116,7 @@ int main(int argc, char **argv_orig, char **envp) { if (read(fr_fd, buf, 32) >= 16) { - sscanf(buf, "%p", (void**)&prev_target_hash); + sscanf(buf, "%p", (void **)&prev_target_hash); } @@ -2126,7 +2126,6 @@ int main(int argc, char **argv_orig, char **envp) { } - write_setup_file(afl, argc, argv); if (afl->in_place_resume) { @@ -2143,8 +2142,23 @@ int main(int argc, char **argv_orig, char **envp) { snprintf(fn, PATH_MAX, "%s/fastresume.bin", afl->out_dir); if ((fr_fd = open(fn, O_RDONLY)) >= 0) { - OKF("Performing FAST RESUME"); - // fast_resume = 1; + u8 ver_string[8]; + u64 *ver = (u64 *)ver_string; + u64 expect_ver = + afl->shm.cmplog_mode + (sizeof(struct queue_entry) << 1); + + if (read(fr_fd, ver_string, sizeof(ver_string)) != sizeof(ver_string)) + WARNF("Emtpy fastresume.bin, ignoring, cannot perform FAST RESUME"); + else if (expect_ver != *ver) + WARNF( + "Different AFL++ version or feature usage, cannot perform FAST " + "RESUME"); + else { + + OKF("Will perform FAST RESUME"); + fast_resume = 1; + + } } else { @@ -2156,42 +2170,34 @@ int main(int argc, char **argv_orig, char **envp) { } - if (fast_resume) { + read_testcases(afl, NULL); - // XXX + pivot_inputs(afl); - } else { - - read_testcases(afl, NULL); - - pivot_inputs(afl); + if (!afl->timeout_given) { find_timeout(afl); } // only for resumes! - if (!afl->timeout_given) { find_timeout(afl); } // only for resumes! + if (afl->afl_env.afl_tmpdir && !afl->in_place_resume) { - if (afl->afl_env.afl_tmpdir && !afl->in_place_resume) { + char tmpfile[PATH_MAX]; - char tmpfile[PATH_MAX]; + if (afl->file_extension) { - if (afl->file_extension) { + snprintf(tmpfile, PATH_MAX, "%s/.cur_input.%s", afl->tmp_dir, + afl->file_extension); - snprintf(tmpfile, PATH_MAX, "%s/.cur_input.%s", afl->tmp_dir, - afl->file_extension); - - } else { - - snprintf(tmpfile, PATH_MAX, "%s/.cur_input", afl->tmp_dir); + } else { - } + snprintf(tmpfile, PATH_MAX, "%s/.cur_input", afl->tmp_dir); - /* there is still a race condition here, but well ... */ - if (access(tmpfile, F_OK) != -1) { + } - FATAL( - "AFL_TMPDIR already has an existing temporary input file: %s - if " - "this is not from another instance, then just remove the file.", - tmpfile); + /* there is still a race condition here, but well ... */ + if (access(tmpfile, F_OK) != -1) { - } + FATAL( + "AFL_TMPDIR already has an existing temporary input file: %s - if " + "this is not from another instance, then just remove the file.", + tmpfile); } @@ -2474,7 +2480,7 @@ int main(int argc, char **argv_orig, char **envp) { } - OKF("Cmplog forkserver successfully started"); + OKF("CMPLOG forkserver successfully started"); } @@ -2512,29 +2518,102 @@ int main(int argc, char **argv_orig, char **envp) { dedup_extras(afl); if (afl->extras_cnt) { OKF("Loaded a total of %u extras.", afl->extras_cnt); } - // after we have the correct bitmap size we can read the bitmap -B option - // and set the virgin maps - if (afl->in_bitmap) { + if (unlikely(fast_resume)) { - read_bitmap(afl->in_bitmap, afl->virgin_bits, afl->fsrv.map_size); + u64 resume_start = get_cur_time_us(); + // if we get here then we should abort on errors + ck_read(fr_fd, afl->virgin_bits, afl->fsrv.map_size, "virgin_bits"); + ck_read(fr_fd, afl->virgin_tmout, afl->fsrv.map_size, "virgin_tmout"); + ck_read(fr_fd, afl->virgin_crash, afl->fsrv.map_size, "virgin_crash"); + ck_read(fr_fd, afl->var_bytes, afl->fsrv.map_size, "var_bytes"); - } else { + u8 res[1] = {0}; + u8 *o_start = (u8 *)&(afl->queue_buf[0]->colorized); + u8 *o_end = (u8 *)&(afl->queue_buf[0]->mother); + u32 r = 8 + afl->fsrv.map_size * 4; + u32 q_len = o_end - o_start; + u32 m_len = (afl->fsrv.map_size >> 3); + struct queue_entry *q; - memset(afl->virgin_bits, 255, map_size); + for (u32 i = 0; i < afl->queued_items; i++) { - } + q = afl->queue_buf[i]; + ck_read(fr_fd, (u8 *)&(q->colorized), q_len, "queue data"); + ck_read(fr_fd, res, 1, "check map"); + if (res[0]) { - memset(afl->virgin_tmout, 255, map_size); - memset(afl->virgin_crash, 255, map_size); + q->trace_mini = ck_alloc(m_len); + ck_read(fr_fd, q->trace_mini, m_len, "trace_mini"); + r += q_len + m_len + 1; - if (likely(!afl->afl_env.afl_no_startup_calibration)) { + } else { + + r += q_len + 1; + + } + + afl->total_bitmap_size += q->bitmap_size; + ++afl->total_bitmap_entries; + update_bitmap_score(afl, q); + + if (q->was_fuzzed) { --afl->pending_not_fuzzed; } + + if (q->disabled) { + + if (!q->was_fuzzed) { --afl->pending_not_fuzzed; } + --afl->active_items; + + } - perform_dry_run(afl); + if (q->var_behavior) { ++afl->queued_variable; } + if (q->favored) { + + ++afl->queued_favored; + if (!q->was_fuzzed) { ++afl->pending_favored; } + + } + + } + + u8 buf[4]; + if (read(fr_fd, buf, 3) > 0) { + + FATAL("invalid trailing data in fastresume.bin"); + + } + + OKF("Successfully loaded fastresume.bin (%u bytes)!", r); + close(fr_fd); + afl->reinit_table = 1; + update_calibration_time(afl, &resume_start); } else { - ACTF("skipping initial seed calibration due option override!"); - usleep(1000); + // after we have the correct bitmap size we can read the bitmap -B option + // and set the virgin maps + if (afl->in_bitmap) { + + read_bitmap(afl->in_bitmap, afl->virgin_bits, afl->fsrv.map_size); + + } else { + + memset(afl->virgin_bits, 255, map_size); + + } + + memset(afl->virgin_tmout, 255, map_size); + memset(afl->virgin_crash, 255, map_size); + + if (likely(!afl->afl_env.afl_no_startup_calibration)) { + + perform_dry_run(afl); + + } else { + + ACTF("Skipping initial seed calibration due option override!"); + usleep(1000); + + } } @@ -3125,6 +3204,63 @@ stop_fuzzing: fclose(afl->fsrv.det_plot_file); #endif + /* create fastresume.bin */ + u8 fr[PATH_MAX]; + snprintf(fr, PATH_MAX, "%s/fastresume.bin", afl->out_dir); + if ((fr_fd = open(fr, O_WRONLY | O_TRUNC | O_CREAT, DEFAULT_PERMISSION)) >= + 0) { + + u8 ver_string[8]; + u32 w = 0; + u64 *ver = (u64 *)ver_string; + *ver = afl->shm.cmplog_mode + (sizeof(struct queue_entry) << 1); + + w += write(fr_fd, ver_string, sizeof(ver_string)); + + w += write(fr_fd, afl->virgin_bits, afl->fsrv.map_size); + w += write(fr_fd, afl->virgin_tmout, afl->fsrv.map_size); + w += write(fr_fd, afl->virgin_crash, afl->fsrv.map_size); + w += write(fr_fd, afl->var_bytes, afl->fsrv.map_size); + + u8 on[1] = {1}, off[1] = {0}; + u8 *o_start = (u8 *)&(afl->queue_buf[0]->colorized); + u8 *o_end = (u8 *)&(afl->queue_buf[0]->mother); + u32 q_len = o_end - o_start; + u32 m_len = (afl->fsrv.map_size >> 3); + struct queue_entry *q; + + afl->pending_not_fuzzed = afl->queued_items; + afl->active_items = afl->queued_items; + + for (u32 i = 0; i < afl->queued_items; i++) { + + q = afl->queue_buf[i]; + ck_write(fr_fd, (u8 *)&(q->colorized), q_len, "queue data"); + if (!q->trace_mini) { + + ck_write(fr_fd, off, 1, "no_mini"); + w += q_len + 1; + + } else { + + ck_write(fr_fd, on, 1, "yes_mini"); + ck_write(fr_fd, q->trace_mini, m_len, "trace_mini"); + w += q_len + m_len + 1; + + } + + } + + close(fr_fd); + afl->var_byte_count = count_bytes(afl, afl->var_bytes); + OKF("Written fastresume.bin with %u bytes!", w); + + } else { + + WARNF("Could not create fastresume.bin"); + + } + destroy_queue(afl); destroy_extras(afl); destroy_custom_mutators(afl); -- cgit 1.4.1 From de176a10bc225e5fa2e7dafdd784f568cc1ca5e7 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 17 Jun 2024 09:31:36 +0200 Subject: nit --- src/afl-fuzz.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index a7ddef6e..cc58b4d1 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -960,7 +960,11 @@ int main(int argc, char **argv_orig, char **envp) { } - if (afl->fsrv.mem_limit < 5) { FATAL("Dangerously low value of -m"); } + if (afl->fsrv.mem_limit && afl->fsrv.mem_limit < 5) { + + FATAL("Dangerously low value of -m"); + + } if (sizeof(rlim_t) == 4 && afl->fsrv.mem_limit > 2000) { -- cgit 1.4.1 From 304e84502d9bd8a5ac33328ceb63235f42b887ad Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 18 Jun 2024 15:24:38 +0200 Subject: fast resume option --- docs/env_variables.md | 2 ++ include/afl-fuzz.h | 2 +- include/envs.h | 2 +- src/afl-fuzz-state.c | 7 +++++++ src/afl-fuzz.c | 12 ++++++++++-- 5 files changed, 21 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/docs/env_variables.md b/docs/env_variables.md index 22e0ce0f..6db31df0 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -514,6 +514,8 @@ checks or alter some of the more exotic semantics of the tool: - `AFL_NO_SNAPSHOT` will advise afl-fuzz not to use the snapshot feature if the snapshot lkm is loaded. + - `AFL_NO_FASTRESUME` will not try to read or write a fast resume file. + - Setting `AFL_NO_UI` inhibits the UI altogether and just periodically prints some basic stats. This behavior is also automatically triggered when the output from afl-fuzz is redirected to a file or to a pipe. diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 3f3d6da0..6d03a74c 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -454,7 +454,7 @@ typedef struct afl_env_vars { afl_no_startup_calibration, afl_no_warn_instability, afl_post_process_keep_original, afl_crashing_seeds_as_new_crash, afl_final_sync, afl_ignore_seed_problems, afl_disable_redundant, - afl_sha1_filenames, afl_no_sync; + afl_sha1_filenames, afl_no_sync, afl_no_fastresume; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_preload, diff --git a/include/envs.h b/include/envs.h index 45b080cb..928f4185 100644 --- a/include/envs.h +++ b/include/envs.h @@ -115,7 +115,7 @@ static char *afl_environment_variables[] = { "AFL_TRACE_PC", "AFL_USE_ASAN", "AFL_USE_MSAN", "AFL_USE_TRACE_PC", "AFL_USE_UBSAN", "AFL_USE_TSAN", "AFL_USE_CFISAN", "AFL_USE_LSAN", "AFL_WINE_PATH", "AFL_NO_SNAPSHOT", "AFL_EXPAND_HAVOC_NOW", "AFL_USE_FASAN", - "AFL_USE_QASAN", "AFL_PRINT_FILENAMES", "AFL_PIZZA_MODE", NULL + "AFL_USE_QASAN", "AFL_PRINT_FILENAMES", "AFL_PIZZA_MODE", "AFL_NO_FASTRESUME", NULL }; diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index fbe6d32a..dd684a19 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -286,6 +286,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_no_sync = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_NO_FASTRESUME", + + afl_environment_variable_len)) { + + afl->afl_env.afl_no_fastresume = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_CUSTOM_MUTATOR_ONLY", afl_environment_variable_len)) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index d8be5721..8fd3a407 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -335,6 +335,7 @@ static void usage(u8 *argv0, int more_help) { "AFL_STATSD_PORT: change default statsd port (default: 8125)\n" "AFL_STATSD_TAGS_FLAVOR: set statsd tags format (default: disable tags)\n" " suported formats: dogstatsd, librato, signalfx, influxdb\n" + "AFL_NO_FASTRESUME: do not read or write a fast resume file\n" "AFL_NO_SYNC: disables all syncing\n" "AFL_SYNC_TIME: sync time between fuzzing instances (in minutes)\n" "AFL_FINAL_SYNC: sync a final time when exiting (will delay the exit!)\n" @@ -2107,7 +2108,7 @@ int main(int argc, char **argv_orig, char **envp) { u64 prev_target_hash = 0; s32 fast_resume = 0, fr_fd = -1; - if (afl->in_place_resume) { + if (afl->in_place_resume && !afl->afl_env.afl_no_fastresume) { u8 fn[PATH_MAX], buf[32]; snprintf(fn, PATH_MAX, "%s/target_hash", afl->out_dir); @@ -2128,7 +2129,7 @@ int main(int argc, char **argv_orig, char **envp) { write_setup_file(afl, argc, argv); - if (afl->in_place_resume) { + if (afl->in_place_resume && !afl->afl_env.afl_no_fastresume) { u64 target_hash = get_binary_hash(afl->fsrv.target_path); @@ -2166,6 +2167,10 @@ int main(int argc, char **argv_orig, char **envp) { } + // If the fast resume file is not valid we will be unable to start, so + // we remove the file but keep the file descriptor open. + unlink(fn); + } } @@ -3204,9 +3209,11 @@ stop_fuzzing: fclose(afl->fsrv.det_plot_file); #endif + if (!afl->afl_env.afl_no_fastresume) { /* create fastresume.bin */ u8 fr[PATH_MAX]; snprintf(fr, PATH_MAX, "%s/fastresume.bin", afl->out_dir); + ACTF("Writing %s ...", fr); if ((fr_fd = open(fr, O_WRONLY | O_TRUNC | O_CREAT, DEFAULT_PERMISSION)) >= 0) { @@ -3260,6 +3267,7 @@ stop_fuzzing: WARNF("Could not create fastresume.bin"); } + } destroy_queue(afl); destroy_extras(afl); -- cgit 1.4.1 From b8568034f0c120ab8500c03ed4982d641eaa88fb Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 18 Jun 2024 15:42:34 +0200 Subject: code format and changelog --- docs/Changelog.md | 5 +++ include/envs.h | 3 +- src/afl-fuzz-queue.c | 4 ++- src/afl-fuzz.c | 90 +++++++++++++++++++++++++++------------------------- 4 files changed, 56 insertions(+), 46 deletions(-) (limited to 'src') diff --git a/docs/Changelog.md b/docs/Changelog.md index 1590b2df..f146534f 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -4,6 +4,11 @@ release of the tool. See README.md for the general instruction manual. ### Version ++4.22a (dev) + - afl-fuzz: + - fastresume feature added. if you abort fuzzing and resume fuzzing + with `-i -` or `AFL_AUTORESUME=1` and the target binary has not changed + then a dump will be loaded and the calibration phase skipped. + to disable this feature set `AFL_NO_FASTRESUME=1` - frida_mode: - AFL_FRIDA_PERSISTENT_ADDR can now be be any reachable address not just a function entry diff --git a/include/envs.h b/include/envs.h index 928f4185..ef522ab4 100644 --- a/include/envs.h +++ b/include/envs.h @@ -115,7 +115,8 @@ static char *afl_environment_variables[] = { "AFL_TRACE_PC", "AFL_USE_ASAN", "AFL_USE_MSAN", "AFL_USE_TRACE_PC", "AFL_USE_UBSAN", "AFL_USE_TSAN", "AFL_USE_CFISAN", "AFL_USE_LSAN", "AFL_WINE_PATH", "AFL_NO_SNAPSHOT", "AFL_EXPAND_HAVOC_NOW", "AFL_USE_FASAN", - "AFL_USE_QASAN", "AFL_PRINT_FILENAMES", "AFL_PIZZA_MODE", "AFL_NO_FASTRESUME", NULL + "AFL_USE_QASAN", "AFL_PRINT_FILENAMES", "AFL_PIZZA_MODE", + "AFL_NO_FASTRESUME", NULL }; diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index a28172f9..6069f5b9 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -401,7 +401,9 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) { } else { - if (unlink(fn)) { /*PFATAL("Unable to remove '%s'", fn);*/ } + if (unlink(fn)) { /*PFATAL("Unable to remove '%s'", fn);*/ + + } } diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 2bfbee15..a09a53ec 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -3214,64 +3214,66 @@ stop_fuzzing: #endif if (!afl->afl_env.afl_no_fastresume) { - /* create fastresume.bin */ - u8 fr[PATH_MAX]; - snprintf(fr, PATH_MAX, "%s/fastresume.bin", afl->out_dir); - ACTF("Writing %s ...", fr); - if ((fr_fd = open(fr, O_WRONLY | O_TRUNC | O_CREAT, DEFAULT_PERMISSION)) >= - 0) { - - u8 ver_string[8]; - u32 w = 0; - u64 *ver = (u64 *)ver_string; - *ver = afl->shm.cmplog_mode + (sizeof(struct queue_entry) << 1); - - w += write(fr_fd, ver_string, sizeof(ver_string)); - - w += write(fr_fd, afl->virgin_bits, afl->fsrv.map_size); - w += write(fr_fd, afl->virgin_tmout, afl->fsrv.map_size); - w += write(fr_fd, afl->virgin_crash, afl->fsrv.map_size); - w += write(fr_fd, afl->var_bytes, afl->fsrv.map_size); - - u8 on[1] = {1}, off[1] = {0}; - u8 *o_start = (u8 *)&(afl->queue_buf[0]->colorized); - u8 *o_end = (u8 *)&(afl->queue_buf[0]->mother); - u32 q_len = o_end - o_start; - u32 m_len = (afl->fsrv.map_size >> 3); - struct queue_entry *q; - afl->pending_not_fuzzed = afl->queued_items; - afl->active_items = afl->queued_items; + /* create fastresume.bin */ + u8 fr[PATH_MAX]; + snprintf(fr, PATH_MAX, "%s/fastresume.bin", afl->out_dir); + ACTF("Writing %s ...", fr); + if ((fr_fd = open(fr, O_WRONLY | O_TRUNC | O_CREAT, DEFAULT_PERMISSION)) >= + 0) { - for (u32 i = 0; i < afl->queued_items; i++) { + u8 ver_string[8]; + u32 w = 0; + u64 *ver = (u64 *)ver_string; + *ver = afl->shm.cmplog_mode + (sizeof(struct queue_entry) << 1); - q = afl->queue_buf[i]; - ck_write(fr_fd, (u8 *)&(q->colorized), q_len, "queue data"); - if (!q->trace_mini) { + w += write(fr_fd, ver_string, sizeof(ver_string)); - ck_write(fr_fd, off, 1, "no_mini"); - w += q_len + 1; + w += write(fr_fd, afl->virgin_bits, afl->fsrv.map_size); + w += write(fr_fd, afl->virgin_tmout, afl->fsrv.map_size); + w += write(fr_fd, afl->virgin_crash, afl->fsrv.map_size); + w += write(fr_fd, afl->var_bytes, afl->fsrv.map_size); - } else { + u8 on[1] = {1}, off[1] = {0}; + u8 *o_start = (u8 *)&(afl->queue_buf[0]->colorized); + u8 *o_end = (u8 *)&(afl->queue_buf[0]->mother); + u32 q_len = o_end - o_start; + u32 m_len = (afl->fsrv.map_size >> 3); + struct queue_entry *q; + + afl->pending_not_fuzzed = afl->queued_items; + afl->active_items = afl->queued_items; - ck_write(fr_fd, on, 1, "yes_mini"); - ck_write(fr_fd, q->trace_mini, m_len, "trace_mini"); - w += q_len + m_len + 1; + for (u32 i = 0; i < afl->queued_items; i++) { + + q = afl->queue_buf[i]; + ck_write(fr_fd, (u8 *)&(q->colorized), q_len, "queue data"); + if (!q->trace_mini) { + + ck_write(fr_fd, off, 1, "no_mini"); + w += q_len + 1; + + } else { + + ck_write(fr_fd, on, 1, "yes_mini"); + ck_write(fr_fd, q->trace_mini, m_len, "trace_mini"); + w += q_len + m_len + 1; + + } } - } + close(fr_fd); + afl->var_byte_count = count_bytes(afl, afl->var_bytes); + OKF("Written fastresume.bin with %u bytes!", w); - close(fr_fd); - afl->var_byte_count = count_bytes(afl, afl->var_bytes); - OKF("Written fastresume.bin with %u bytes!", w); + } else { - } else { + WARNF("Could not create fastresume.bin"); - WARNF("Could not create fastresume.bin"); + } } - } destroy_queue(afl); destroy_extras(afl); -- cgit 1.4.1 From 8fcca6fb410a6ece1a4cd2eb8a2cdeed4d4d9865 Mon Sep 17 00:00:00 2001 From: "Christian Holler (:decoder)" Date: Wed, 19 Jun 2024 12:36:58 +0200 Subject: Collect persistent coverage data and dump it at the end of the run With CODE_COVERAGE builds, we need to collect the coverage data of each iteration in a persistant buffer that has the same size as the regular trace buffer used for fuzzing. We dump this information at the end of the run and when combined with pointer data and module info, this can be used to calculate code coverage. --- include/forkserver.h | 4 ++++ src/afl-forkserver.c | 8 ++++++++ src/afl-fuzz-run.c | 21 +++++++++++++++++++++ src/afl-fuzz.c | 22 ++++++++++++++++++++++ 4 files changed, 55 insertions(+) (limited to 'src') diff --git a/include/forkserver.h b/include/forkserver.h index 593e34a2..3fd813a4 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -206,6 +206,10 @@ typedef struct afl_forkserver { s32 nyx_log_fd; #endif +#ifdef __AFL_CODE_COVERAGE + u8 *persistent_trace_bits; /* Persistent copy of bitmap */ +#endif + } afl_forkserver_t; typedef enum fsrv_run_result { diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 71d8570d..a998c10f 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -252,6 +252,10 @@ void afl_fsrv_init(afl_forkserver_t *fsrv) { fsrv->uses_crash_exitcode = false; fsrv->uses_asan = false; +#ifdef __AFL_CODE_COVERAGE + fsrv->persistent_trace_bits = NULL; +#endif + fsrv->init_child_func = fsrv_exec_child; list_append(&fsrv_list, fsrv); @@ -278,6 +282,10 @@ void afl_fsrv_init_dup(afl_forkserver_t *fsrv_to, afl_forkserver_t *from) { fsrv_to->fsrv_kill_signal = from->fsrv_kill_signal; fsrv_to->debug = from->debug; +#ifdef __AFL_CODE_COVERAGE + fsrv_to->persistent_trace_bits = from->persistent_trace_bits; +#endif + // These are forkserver specific. fsrv_to->out_dir_fd = -1; fsrv_to->child_pid = -1; diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 6a0da6ab..c234fc42 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -60,6 +60,27 @@ fuzz_run_target(afl_state_t *afl, afl_forkserver_t *fsrv, u32 timeout) { fsrv_run_result_t res = afl_fsrv_run_target(fsrv, timeout, &afl->stop_soon); +#ifdef __AFL_CODE_COVERAGE + if (unlikely(!fsrv->persistent_trace_bits)) { + + // On the first run, we allocate the persistent map to collect coverage. + fsrv->persistent_trace_bits = (u8 *)malloc(fsrv->map_size); + memset(fsrv->persistent_trace_bits, 0, fsrv->map_size); + + } + + for (u32 i = 0; i < fsrv->map_size; ++i) { + + if (fsrv->persistent_trace_bits[i] != 255 && fsrv->trace_bits[i]) { + + fsrv->persistent_trace_bits[i]++; + + } + + } + +#endif + /* If post_run() function is defined in custom mutator, the function will be called each time after AFL++ executes the target program. */ diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index a09a53ec..0209e74f 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -3130,6 +3130,28 @@ stop_fuzzing: write_bitmap(afl); save_auto(afl); + #ifdef __AFL_CODE_COVERAGE + if (afl->fsrv.persistent_trace_bits) { + + char cfn[4096]; + snprintf(cfn, sizeof(cfn), "%s/covmap.dump", afl->out_dir); + + FILE *cov_fd; + if ((cov_fd = fopen(cfn, "w")) == NULL) { + + PFATAL("could not create '%s'", cfn); + + } + + // Write the real map size, as the map size must exactly match the pointer + // map in length. + fwrite(afl->fsrv.persistent_trace_bits, 1, afl->fsrv.real_map_size, cov_fd); + fclose(cov_fd); + + } + + #endif + if (afl->pizza_is_served) { SAYF(CURSOR_SHOW cLRD "\n\n+++ Baking aborted %s +++\n" cRST, -- cgit 1.4.1 From 2fbc0aefb1b68c5c580c9861541dd7a2080d2f2f Mon Sep 17 00:00:00 2001 From: killerra <25255685+killerra@users.noreply.github.com> Date: Thu, 20 Jun 2024 00:10:40 +0200 Subject: Auto disable memory limits for FASAN --- src/afl-fuzz.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src') diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 0209e74f..0ae12fc1 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1873,6 +1873,13 @@ int main(int argc, char **argv_orig, char **envp) { OKF("Using Frida Address Sanitizer Mode"); + if (afl->fsrv.mem_limit) { + + WARNF("in the Frida Address Sanitizer Mode we disable all memory limits"); + afl->fsrv.mem_limit = 0; + + } + fasan_check_afl_preload(afl_preload); setenv("ASAN_OPTIONS", "detect_leaks=false", 1); -- cgit 1.4.1 From ecb5854be08fa978be3320c1f8333f6cc3261fec Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 21 Jun 2024 14:40:16 +0200 Subject: add zlib compression for fast resume --- GNUmakefile | 8 ++++ docs/Changelog.md | 1 + include/debug.h | 2 - src/afl-fuzz.c | 135 +++++++++++++++++++++++++++++++++++++++++++----------- 4 files changed, 116 insertions(+), 30 deletions(-) (limited to 'src') diff --git a/GNUmakefile b/GNUmakefile index e79d3f83..9f862120 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -441,6 +441,14 @@ test_shm: @echo "[-] shmat seems not to be working, switching to mmap implementation" endif +ifeq "$(shell echo '$(HASH)include @int main() {return 0; }' | tr @ '\n' | $(CC) $(CFLAGS) -Werror -x c - -lz -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + override SPECIAL_PERFORMANCE += -DHAVE_ZLIB + override LDFLAGS += -lz + $(info [+] ZLIB detected) +else + $(info [!] Warning: no ZLIB detected) +endif + .PHONY: test_python ifeq "$(PYTHON_OK)" "1" test_python: diff --git a/docs/Changelog.md b/docs/Changelog.md index f146534f..c6266e86 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -9,6 +9,7 @@ with `-i -` or `AFL_AUTORESUME=1` and the target binary has not changed then a dump will be loaded and the calibration phase skipped. to disable this feature set `AFL_NO_FASTRESUME=1` + zlib compression is used if zlib is found at compile time - frida_mode: - AFL_FRIDA_PERSISTENT_ADDR can now be be any reachable address not just a function entry diff --git a/include/debug.h b/include/debug.h index 5496135c..c66d0334 100644 --- a/include/debug.h +++ b/include/debug.h @@ -409,8 +409,6 @@ static inline const char *colorfilter(const char *x) { \ } while (1); \ \ - \ - \ } while (0) #define ck_read(fd, buf, len, fn) \ diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 0ae12fc1..b53a9a2d 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -36,6 +36,67 @@ #include #include #endif +#ifdef HAVE_ZLIB + + #define ck_gzread(fd, buf, len, fn) \ + do { \ + \ + s32 _len = (s32)(len); \ + s32 _res = gzread(fd, buf, _len); \ + if (_res != _len) RPFATAL(_res, "Short read from %s", fn); \ + \ + } while (0) + + #define ck_gzwrite(fd, buf, len, fn) \ + do { \ + \ + if (len <= 0) break; \ + s32 _written = 0, _off = 0, _len = (s32)(len); \ + \ + do { \ + \ + s32 _res = gzwrite(fd, (buf) + _off, _len); \ + if (_res != _len && (_res > 0 && _written + _res != _len)) { \ + \ + if (_res > 0) { \ + \ + _written += _res; \ + _len -= _res; \ + _off += _res; \ + \ + } else { \ + \ + RPFATAL(_res, "Short write to %s (%d of %d bytes)", fn, _res, \ + _len); \ + \ + } \ + \ + } else { \ + \ + break; \ + \ + } \ + \ + } while (1); \ + \ + \ + \ + } while (0) + + #include + #define ZLIBOPEN gzopen + #define ZLIBREAD ck_gzread + #define NZLIBREAD gzread + #define ZLIBWRITE ck_gzwrite + #define ZLIBCLOSE gzclose + #define ZLIB_EXTRA "9" +#else + #define ZLIBOPEN open + #define NZLIBREAD read + #define ZLIBREAD ck_read + #define ZLIBWRITE ck_write + #define ZLIBCLOSE close +#endif #ifdef __APPLE__ #include @@ -1875,7 +1936,9 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->fsrv.mem_limit) { - WARNF("in the Frida Address Sanitizer Mode we disable all memory limits"); + WARNF( + "in the Frida Address Sanitizer Mode we disable all memory " + "limits"); afl->fsrv.mem_limit = 0; } @@ -2117,22 +2180,27 @@ int main(int argc, char **argv_orig, char **envp) { check_binary(afl, argv[optind]); u64 prev_target_hash = 0; - s32 fast_resume = 0, fr_fd = -1; + s32 fast_resume = 0; + #ifdef HAVE_ZLIB + gzFile fr_fd = NULL; + #else + s32 fr_fd = -1; + #endif if (afl->in_place_resume && !afl->afl_env.afl_no_fastresume) { u8 fn[PATH_MAX], buf[32]; snprintf(fn, PATH_MAX, "%s/target_hash", afl->out_dir); - fr_fd = open(fn, O_RDONLY); - if (fr_fd >= 0) { + s32 fd = open(fn, O_RDONLY); + if (fd >= 0) { - if (read(fr_fd, buf, 32) >= 16) { + if (read(fd, buf, 32) >= 16) { sscanf(buf, "%p", (void **)&prev_target_hash); } - close(fr_fd); + close(fd); } @@ -2152,14 +2220,21 @@ int main(int argc, char **argv_orig, char **envp) { u8 fn[PATH_MAX]; snprintf(fn, PATH_MAX, "%s/fastresume.bin", afl->out_dir); + #ifdef HAVE_ZLIB + if ((fr_fd = ZLIBOPEN(fn, "rb")) != NULL) { + + #else if ((fr_fd = open(fn, O_RDONLY)) >= 0) { + #endif + u8 ver_string[8]; u64 *ver = (u64 *)ver_string; u64 expect_ver = afl->shm.cmplog_mode + (sizeof(struct queue_entry) << 1); - if (read(fr_fd, ver_string, sizeof(ver_string)) != sizeof(ver_string)) + if (NZLIBREAD(fr_fd, ver_string, sizeof(ver_string)) != + sizeof(ver_string)) WARNF("Emtpy fastresume.bin, ignoring, cannot perform FAST RESUME"); else if (expect_ver != *ver) WARNF( @@ -2538,10 +2613,10 @@ int main(int argc, char **argv_orig, char **envp) { u64 resume_start = get_cur_time_us(); // if we get here then we should abort on errors - ck_read(fr_fd, afl->virgin_bits, afl->fsrv.map_size, "virgin_bits"); - ck_read(fr_fd, afl->virgin_tmout, afl->fsrv.map_size, "virgin_tmout"); - ck_read(fr_fd, afl->virgin_crash, afl->fsrv.map_size, "virgin_crash"); - ck_read(fr_fd, afl->var_bytes, afl->fsrv.map_size, "var_bytes"); + ZLIBREAD(fr_fd, afl->virgin_bits, afl->fsrv.map_size, "virgin_bits"); + ZLIBREAD(fr_fd, afl->virgin_tmout, afl->fsrv.map_size, "virgin_tmout"); + ZLIBREAD(fr_fd, afl->virgin_crash, afl->fsrv.map_size, "virgin_crash"); + ZLIBREAD(fr_fd, afl->var_bytes, afl->fsrv.map_size, "var_bytes"); u8 res[1] = {0}; u8 *o_start = (u8 *)&(afl->queue_buf[0]->colorized); @@ -2554,12 +2629,12 @@ int main(int argc, char **argv_orig, char **envp) { for (u32 i = 0; i < afl->queued_items; i++) { q = afl->queue_buf[i]; - ck_read(fr_fd, (u8 *)&(q->colorized), q_len, "queue data"); - ck_read(fr_fd, res, 1, "check map"); + ZLIBREAD(fr_fd, (u8 *)&(q->colorized), q_len, "queue data"); + ZLIBREAD(fr_fd, res, 1, "check map"); if (res[0]) { q->trace_mini = ck_alloc(m_len); - ck_read(fr_fd, q->trace_mini, m_len, "trace_mini"); + ZLIBREAD(fr_fd, q->trace_mini, m_len, "trace_mini"); r += q_len + m_len + 1; } else { @@ -2592,14 +2667,14 @@ int main(int argc, char **argv_orig, char **envp) { } u8 buf[4]; - if (read(fr_fd, buf, 3) > 0) { + if (NZLIBREAD(fr_fd, buf, 3) > 0) { FATAL("invalid trailing data in fastresume.bin"); } OKF("Successfully loaded fastresume.bin (%u bytes)!", r); - close(fr_fd); + ZLIBCLOSE(fr_fd); afl->reinit_table = 1; update_calibration_time(afl, &resume_start); @@ -3248,20 +3323,24 @@ stop_fuzzing: u8 fr[PATH_MAX]; snprintf(fr, PATH_MAX, "%s/fastresume.bin", afl->out_dir); ACTF("Writing %s ...", fr); + #ifdef HAVE_ZLIB + if ((fr_fd = ZLIBOPEN(fr, "wb9")) != NULL) { + + #else if ((fr_fd = open(fr, O_WRONLY | O_TRUNC | O_CREAT, DEFAULT_PERMISSION)) >= - 0) { + #endif u8 ver_string[8]; u32 w = 0; u64 *ver = (u64 *)ver_string; *ver = afl->shm.cmplog_mode + (sizeof(struct queue_entry) << 1); - w += write(fr_fd, ver_string, sizeof(ver_string)); - - w += write(fr_fd, afl->virgin_bits, afl->fsrv.map_size); - w += write(fr_fd, afl->virgin_tmout, afl->fsrv.map_size); - w += write(fr_fd, afl->virgin_crash, afl->fsrv.map_size); - w += write(fr_fd, afl->var_bytes, afl->fsrv.map_size); + ZLIBWRITE(fr_fd, ver_string, sizeof(ver_string), "ver_string"); + ZLIBWRITE(fr_fd, afl->virgin_bits, afl->fsrv.map_size, "virgin_bits"); + ZLIBWRITE(fr_fd, afl->virgin_tmout, afl->fsrv.map_size, "virgin_tmout"); + ZLIBWRITE(fr_fd, afl->virgin_crash, afl->fsrv.map_size, "virgin_crash"); + ZLIBWRITE(fr_fd, afl->var_bytes, afl->fsrv.map_size, "var_bytes"); + w += sizeof(ver_string) + afl->fsrv.map_size * 4; u8 on[1] = {1}, off[1] = {0}; u8 *o_start = (u8 *)&(afl->queue_buf[0]->colorized); @@ -3276,23 +3355,23 @@ stop_fuzzing: for (u32 i = 0; i < afl->queued_items; i++) { q = afl->queue_buf[i]; - ck_write(fr_fd, (u8 *)&(q->colorized), q_len, "queue data"); + ZLIBWRITE(fr_fd, (u8 *)&(q->colorized), q_len, "queue data"); if (!q->trace_mini) { - ck_write(fr_fd, off, 1, "no_mini"); + ZLIBWRITE(fr_fd, off, 1, "no_mini"); w += q_len + 1; } else { - ck_write(fr_fd, on, 1, "yes_mini"); - ck_write(fr_fd, q->trace_mini, m_len, "trace_mini"); + ZLIBWRITE(fr_fd, on, 1, "yes_mini"); + ZLIBWRITE(fr_fd, q->trace_mini, m_len, "trace_mini"); w += q_len + m_len + 1; } } - close(fr_fd); + ZLIBCLOSE(fr_fd); afl->var_byte_count = count_bytes(afl, afl->var_bytes); OKF("Written fastresume.bin with %u bytes!", w); -- cgit 1.4.1 From ac5815d994fe8ff151e0f13088891acc506662ed Mon Sep 17 00:00:00 2001 From: Kuan-Wei Chiu Date: Mon, 24 Jun 2024 05:51:55 +0800 Subject: Optimize bit counting using __builtin_popcount Use the __builtin_popcount intrinsic to optimize the bit counting function if the compiler supports it. This change replaces the manual bit counting algorithm with the more efficient built-in function, which leverages hardware support on compatible processors. This modification ensures that the code remains backward-compatible by falling back to the original implementation when __builtin_popcount is not available. --- include/afl-fuzz.h | 4 ++++ src/afl-fuzz-bitmap.c | 4 ++++ 2 files changed, 8 insertions(+) (limited to 'src') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 6d03a74c..45600698 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -116,6 +116,10 @@ #include #endif +#ifndef __has_builtin + #define __has_builtin(x) 0 +#endif + #undef LIST_FOREACH /* clashes with FreeBSD */ #include "list.h" #ifndef SIMPLE_FILES diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 03bc5d6c..405d2dd6 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -75,9 +75,13 @@ u32 count_bits(afl_state_t *afl, u8 *mem) { } +#if __has_builtin(__builtin_popcount) + ret += __builtin_popcount(v); +#else v -= ((v >> 1) & 0x55555555); v = (v & 0x33333333) + ((v >> 2) & 0x33333333); ret += (((v + (v >> 4)) & 0xF0F0F0F) * 0x01010101) >> 24; +#endif } -- cgit 1.4.1 From af47531745ccdc61b5966f213f6cd6cd9030cc2f Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 26 Jun 2024 01:12:04 +0200 Subject: improved seed selection algorithm --- TODO.md | 3 +- custom_mutators/gramatron/json-c | 2 +- docs/Changelog.md | 1 + src/afl-fuzz-bitmap.c | 4 +- src/afl-fuzz-queue.c | 126 +++++++++++++++++++++++++++++++++------ 5 files changed, 112 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/TODO.md b/TODO.md index 6f7505a6..a1431afa 100644 --- a/TODO.md +++ b/TODO.md @@ -2,8 +2,7 @@ ## Must - - fast resume: - use gzopen(fn, "rb"/"wb9-/f/h", gzwrite, gzread and gzclose + - docs: AFL_DISABLE_REDUNDANT (large/slow/LAF) - check for null ptr for xml/curl/g_ string transform functions - hardened_usercopy=0 page_alloc.shuffle=0 - add value_profile but only enable after 15 minutes without finds diff --git a/custom_mutators/gramatron/json-c b/custom_mutators/gramatron/json-c index 11546bfd..af8dd4a3 160000 --- a/custom_mutators/gramatron/json-c +++ b/custom_mutators/gramatron/json-c @@ -1 +1 @@ -Subproject commit 11546bfd07a575c47416924cb98de3d33a4e6424 +Subproject commit af8dd4a307e7b837f9fa2959549548ace4afe08b diff --git a/docs/Changelog.md b/docs/Changelog.md index c6266e86..09ea8cb6 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -10,6 +10,7 @@ then a dump will be loaded and the calibration phase skipped. to disable this feature set `AFL_NO_FASTRESUME=1` zlib compression is used if zlib is found at compile time + - improved seed selection algorithm - frida_mode: - AFL_FRIDA_PERSISTENT_ADDR can now be be any reachable address not just a function entry diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 405d2dd6..97ccd3d3 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -463,7 +463,7 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { if (unlikely(fault == FSRV_RUN_TMOUT && afl->afl_env.afl_ignore_timeouts)) { - if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) { + if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) { classify_counts(&afl->fsrv); u64 cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); @@ -489,7 +489,7 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { /* Generating a hash on every input is super expensive. Bad idea and should only be used for special schedules */ - if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) { + if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) { classify_counts(&afl->fsrv); classified = 1; diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 6069f5b9..999929a1 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -123,7 +123,7 @@ void create_alias_table(afl_state_t *afl) { double weight = 1.0; { // inline does result in a compile error with LTO, weird - if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) { + if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) { u32 hits = afl->n_fuzz[q->n_fuzz_entry]; if (likely(hits)) { weight /= (log10(hits) + 1); } @@ -133,39 +133,127 @@ void create_alias_table(afl_state_t *afl) { if (likely(afl->schedule < RARE)) { double t = q->exec_us / avg_exec_us; + if (likely(t < 0.1)) { // nothing - } else if (likely(t <= 0.25)) + } else if (likely(t <= 0.25)) { + + weight *= 0.95; - weight *= 0.9; - else if (likely(t <= 0.5)) { + } else if (likely(t <= 0.5)) { // nothing - } else if (likely(t < 1.0)) + } else if (likely(t <= 0.75)) { + + weight *= 1.05; + + } else if (likely(t <= 1.0)) { + + weight *= 1.1; + + } else if (likely(t < 1.25)) { + + weight *= 0.2; // WTF ??? makes no sense + + } else if (likely(t <= 1.5)) { + + // nothing + + } else if (likely(t <= 2.0)) { + + weight *= 1.1; + + } else if (likely(t <= 2.5)) { + + } else if (likely(t <= 5.0)) { weight *= 1.15; - else if (unlikely(t > 2.5 && t < 5.0)) + + } else if (likely(t <= 20.0)) { + weight *= 1.1; - // else nothing + // else nothing + + } } double l = q->len / avg_len; - if (likely(l < 0.1)) - weight *= 0.75; - else if (likely(l < 0.25)) - weight *= 1.1; - else if (unlikely(l >= 10)) - weight *= 1.1; + if (likely(l < 0.1)) { + + weight *= 0.5; + + } else if (likely(l <= 0.5)) { + + // nothing + + } else if (likely(l <= 1.25)) { + + weight *= 1.05; + + } else if (likely(l <= 1.75)) { + + // nothing + + } else if (likely(l <= 2.0)) { + + weight *= 0.95; + + } else if (likely(l <= 5.0)) { + + // nothing + + } else if (likely(l <= 10.0)) { + + weight *= 1.05; + + } else { + + weight *= 1.15; + + } double bms = q->bitmap_size / avg_bitmap_size; - if (likely(bms < 0.5)) - weight *= (1.0 + ((bms - 0.5) / 2)); - else if (unlikely(bms > 1.33)) - weight *= 1.1; + if (likely(bms < 0.1)) { + + weight *= 0.01; + + } else if (likely(bms <= 0.25)) { + + weight *= 0.55; + + } else if (likely(bms <= 0.5)) { + + // nothing + + } else if (likely(bms <= 0.75)) { + + weight *= 1.2; + + } else if (likely(bms <= 1.25)) { + + weight *= 1.3; + + } else if (likely(bms <= 1.75)) { + + weight *= 1.25; + + } else if (likely(bms <= 2.0)) { + + // nothing + + } else if (likely(bms <= 2.5)) { + + weight *= 1.3; + + } else { + + weight *= 0.75; + + } if (unlikely(!q->was_fuzzed)) { weight *= 2.5; } if (unlikely(q->fs_redundant)) { weight *= 0.75; } @@ -741,7 +829,7 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { u64 fav_factor; u64 fuzz_p2; - if (likely(afl->schedule >= FAST && afl->schedule < RARE)) { + if (unlikely(afl->schedule >= FAST && afl->schedule < RARE)) { fuzz_p2 = 0; // Skip the fuzz_p2 comparison @@ -777,7 +865,7 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { u64 top_rated_fav_factor; u64 top_rated_fuzz_p2; - if (likely(afl->schedule >= FAST && afl->schedule < RARE)) { + if (unlikely(afl->schedule >= FAST && afl->schedule < RARE)) { top_rated_fuzz_p2 = 0; // Skip the fuzz_p2 comparison -- cgit 1.4.1 From ba7313b521df7a347cd9f96a694cd8caa63b9b41 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 27 Jun 2024 18:51:54 +0200 Subject: AFL_CUSTOM_MUTATOR_LATE_SEND added --- docs/Changelog.md | 2 ++ docs/custom_mutators.md | 5 +++++ docs/env_variables.md | 4 ++++ include/afl-fuzz.h | 5 +++-- include/envs.h | 19 ++++++++++--------- include/forkserver.h | 5 +++++ src/afl-forkserver.c | 10 ++++++++++ src/afl-fuzz-run.c | 12 +++++++++++- src/afl-fuzz-state.c | 7 +++++++ src/afl-fuzz.c | 25 +++++++++++++++++++++++++ 10 files changed, 82 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/docs/Changelog.md b/docs/Changelog.md index 09ea8cb6..c16214e4 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -11,6 +11,8 @@ to disable this feature set `AFL_NO_FASTRESUME=1` zlib compression is used if zlib is found at compile time - improved seed selection algorithm + - added `AFL_CUSTOM_MUTATOR_LATE_SEND=1` to call the custom send() + function after the target has been restarted. - frida_mode: - AFL_FRIDA_PERSISTENT_ADDR can now be be any reachable address not just a function entry diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index b7a7032f..3067ceab 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -198,6 +198,11 @@ def deinit(): # optional for Python This method can be used if you want to send data to the target yourself, e.g. via IPC. This replaces some usage of utils/afl_proxy but requires that you start the target with afl-fuzz. + + Setting `AFL_CUSTOM_MUTATOR_LATE_SEND` will call the afl_custom_fuzz_send() + function after the target has been restarted. (This is needed for e.g. TCP + services.) + Example: [custom_mutators/examples/custom_send.c](../custom_mutators/examples/custom_send.c) - `queue_new_entry` (optional): diff --git a/docs/env_variables.md b/docs/env_variables.md index 6db31df0..eebbcbda 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -368,6 +368,10 @@ checks or alter some of the more exotic semantics of the tool: XML or other highly flexible structured input. For details, see [custom_mutators.md](custom_mutators.md). + - Setting `AFL_CUSTOM_MUTATOR_LATE_SEND` will call the afl_custom_fuzz_send() + function after the target has been restarted. (This is needed for e.g. TCP + services.) + - Setting `AFL_CYCLE_SCHEDULES` will switch to a different schedule every time a cycle is finished. diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 45600698..0f0e45d3 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -449,8 +449,9 @@ extern char *power_names[POWER_SCHEDULES_NUM]; typedef struct afl_env_vars { u8 afl_skip_cpufreq, afl_exit_when_done, afl_no_affinity, afl_skip_bin_check, - afl_dumb_forksrv, afl_import_first, afl_custom_mutator_only, afl_no_ui, - afl_force_ui, afl_i_dont_care_about_missing_crashes, afl_bench_just_one, + afl_dumb_forksrv, afl_import_first, afl_custom_mutator_only, + afl_custom_mutator_late_send, afl_no_ui, afl_force_ui, + afl_i_dont_care_about_missing_crashes, afl_bench_just_one, afl_bench_until_crash, afl_debug_child, afl_autoresume, afl_cal_fast, afl_cycle_schedules, afl_expand_havoc, afl_statsd, afl_cmplog_only_new, afl_exit_on_seed_issues, afl_try_affinity, afl_ignore_problems, diff --git a/include/envs.h b/include/envs.h index ef522ab4..6a0d329b 100644 --- a/include/envs.h +++ b/include/envs.h @@ -24,15 +24,16 @@ static char *afl_environment_variables[] = { "AFL_DUMP_CYCLOMATIC_COMPLEXITY", "AFL_CMPLOG_MAX_LEN", "AFL_COMPCOV_LEVEL", "AFL_CRASH_EXITCODE", "AFL_CRASHING_SEEDS_AS_NEW_CRASH", "AFL_CUSTOM_MUTATOR_LIBRARY", "AFL_CUSTOM_MUTATOR_ONLY", - "AFL_CUSTOM_INFO_PROGRAM", "AFL_CUSTOM_INFO_PROGRAM_ARGV", - "AFL_CUSTOM_INFO_PROGRAM_INPUT", "AFL_CUSTOM_INFO_OUT", "AFL_CXX", - "AFL_CYCLE_SCHEDULES", "AFL_DEBUG", "AFL_DEBUG_CHILD", "AFL_DEBUG_GDB", - "AFL_DEBUG_UNICORN", "AFL_DISABLE_REDUNDANT", "AFL_NO_REDUNDANT", - "AFL_DISABLE_TRIM", "AFL_NO_TRIM", "AFL_DISABLE_LLVM_INSTRUMENTATION", - "AFL_DONT_OPTIMIZE", "AFL_DRIVER_STDERR_DUPLICATE_FILENAME", - "AFL_DUMB_FORKSRV", "AFL_EARLY_FORKSERVER", "AFL_ENTRYPOINT", - "AFL_EXIT_WHEN_DONE", "AFL_EXIT_ON_TIME", "AFL_EXIT_ON_SEED_ISSUES", - "AFL_FAST_CAL", "AFL_FINAL_SYNC", "AFL_FORCE_UI", "AFL_FRIDA_DEBUG_MAPS", + "AFL_CUSTOM_MUTATOR_LATE_SEND", "AFL_CUSTOM_INFO_PROGRAM", + "AFL_CUSTOM_INFO_PROGRAM_ARGV", "AFL_CUSTOM_INFO_PROGRAM_INPUT", + "AFL_CUSTOM_INFO_OUT", "AFL_CXX", "AFL_CYCLE_SCHEDULES", "AFL_DEBUG", + "AFL_DEBUG_CHILD", "AFL_DEBUG_GDB", "AFL_DEBUG_UNICORN", + "AFL_DISABLE_REDUNDANT", "AFL_NO_REDUNDANT", "AFL_DISABLE_TRIM", + "AFL_NO_TRIM", "AFL_DISABLE_LLVM_INSTRUMENTATION", "AFL_DONT_OPTIMIZE", + "AFL_DRIVER_STDERR_DUPLICATE_FILENAME", "AFL_DUMB_FORKSRV", + "AFL_EARLY_FORKSERVER", "AFL_ENTRYPOINT", "AFL_EXIT_WHEN_DONE", + "AFL_EXIT_ON_TIME", "AFL_EXIT_ON_SEED_ISSUES", "AFL_FAST_CAL", + "AFL_FINAL_SYNC", "AFL_FORCE_UI", "AFL_FRIDA_DEBUG_MAPS", "AFL_FRIDA_DRIVER_NO_HOOK", "AFL_FRIDA_EXCLUDE_RANGES", "AFL_FRIDA_INST_CACHE_SIZE", "AFL_FRIDA_INST_COVERAGE_ABSOLUTE", "AFL_FRIDA_INST_COVERAGE_FILE", "AFL_FRIDA_INST_DEBUG_FILE", diff --git a/include/forkserver.h b/include/forkserver.h index 3fd813a4..6c649528 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -210,6 +210,11 @@ typedef struct afl_forkserver { u8 *persistent_trace_bits; /* Persistent copy of bitmap */ #endif + void *custom_data_ptr; + u8 *custom_input; + u32 custom_input_len; + void (*late_send)(void *, const u8 *, size_t); + } afl_forkserver_t; typedef enum fsrv_run_result { diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index a998c10f..cec91f76 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -292,6 +292,9 @@ void afl_fsrv_init_dup(afl_forkserver_t *fsrv_to, afl_forkserver_t *from) { fsrv_to->use_fauxsrv = 0; fsrv_to->last_run_timed_out = 0; + fsrv_to->late_send = from->late_send; + fsrv_to->custom_data_ptr = from->custom_data_ptr; + fsrv_to->init_child_func = from->init_child_func; // Note: do not copy ->add_extra_func or ->persistent_record* @@ -1952,6 +1955,13 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, } + if (unlikely(fsrv->late_send)) { + + fsrv->late_send(fsrv->custom_data_ptr, fsrv->custom_input, + fsrv->custom_input_len); + + } + exec_ms = read_s32_timed(fsrv->fsrv_st_fd, &fsrv->child_status, timeout, stop_soon_p); diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index c234fc42..2f244a1d 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -194,7 +194,17 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { if (el->afl_custom_fuzz_send) { - el->afl_custom_fuzz_send(el->data, *mem, new_size); + if (!afl->afl_env.afl_custom_mutator_late_send) { + + el->afl_custom_fuzz_send(el->data, *mem, new_size); + + } else { + + afl->fsrv.custom_input = *mem; + afl->fsrv.custom_input_len = new_size; + + } + sent = 1; } diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index dd684a19..eead3e50 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -300,6 +300,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_custom_mutator_only = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_CUSTOM_MUTATOR_LATE_SEND", + + afl_environment_variable_len)) { + + afl->afl_env.afl_custom_mutator_late_send = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_CMPLOG_ONLY_NEW", afl_environment_variable_len)) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index b53a9a2d..8a84d447 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2142,6 +2142,31 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->custom_mutators_count && afl->afl_env.afl_custom_mutator_late_send) { + + u32 count_send = 0; + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_fuzz_send) { + + if (count_send) { + + FATAL( + "You can only have one custom send() function if you are using " + "AFL_CUSTOM_MUTATOR_LATE_SEND!"); + + } + + afl->fsrv.late_send = el->afl_custom_fuzz_send; + afl->fsrv.custom_data_ptr = el->data; + count_send = 1; + + } + + }); + + } + if (afl->limit_time_sig > 0 && afl->custom_mutators_count) { if (afl->custom_only) { -- cgit 1.4.1