From 2d9b793dbbe9288a1caa4459c280678179bb46c9 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 4 Jun 2024 14:47:58 +0200 Subject: AFL_NO_SYNC --- docs/Changelog.md | 2 ++ 1 file changed, 2 insertions(+) (limited to 'docs/Changelog.md') diff --git a/docs/Changelog.md b/docs/Changelog.md index ba7eb6a3..1f6a940e 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -10,6 +10,7 @@ three times faster. The reason for this is unknown. - added AFL_DISABLE_REDUNDANT for huge queues - fix AFL_PERSISTENT_RECORD + - added `AFL_NO_SYNC` environment variable that does what you think it does - run custom_post_process after standard trimming - prevent filenames in the queue that have spaces - minor fix for FAST schedules @@ -32,6 +33,7 @@ * afl-showmap - fix memory leak on shmem testcase usage (thanks to @ndrewh) - minor fix to collect coverage -C (thanks to @bet4it) + * libtokencap: script generate_libtoken_dict.sh added by @a-shvedov * enhanced the ASAN configuration -- cgit 1.4.1 From e46c106b890404fbeb2d0e6120510ddf83113da6 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 6 Jun 2024 10:25:19 +0200 Subject: new seed selection algorithm --- docs/Changelog.md | 6 +++++- src/afl-fuzz-queue.c | 59 +++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 50 insertions(+), 15 deletions(-) (limited to 'docs/Changelog.md') diff --git a/docs/Changelog.md b/docs/Changelog.md index 1f6a940e..633e7071 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -8,9 +8,12 @@ - fixed a regression in afl-fuzz that resulted in a 5-10% performace loss do a switch from gettimeofday() to clock_gettime() which should be rather three times faster. The reason for this is unknown. + - new queue selection algorithm based on 2 core years of queue data + analysis. gives a noticable improvement on coverage although the results + seem counterintuitive :-) - added AFL_DISABLE_REDUNDANT for huge queues - - fix AFL_PERSISTENT_RECORD - added `AFL_NO_SYNC` environment variable that does what you think it does + - fix AFL_PERSISTENT_RECORD - run custom_post_process after standard trimming - prevent filenames in the queue that have spaces - minor fix for FAST schedules @@ -33,6 +36,7 @@ * afl-showmap - fix memory leak on shmem testcase usage (thanks to @ndrewh) - minor fix to collect coverage -C (thanks to @bet4it) + * Fixed a shmem mmap bug (that rarely came up on MacOS) * libtokencap: script generate_libtoken_dict.sh added by @a-shvedov * enhanced the ASAN configuration diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 784b377a..d19dd51a 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -60,9 +60,9 @@ inline u32 select_next_queue_entry(afl_state_t *afl) { } -double compute_weight(afl_state_t *afl, struct queue_entry *q, - double avg_exec_us, double avg_bitmap_size, - double avg_top_size) { +inline double compute_weight(afl_state_t *afl, struct queue_entry *q, + double avg_exec_us, double avg_bitmap_size, + double avg_len) { double weight = 1.0; @@ -73,14 +73,45 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q, } - if (likely(afl->schedule < RARE)) { weight *= (avg_exec_us / q->exec_us); } - weight *= (log(q->bitmap_size) / avg_bitmap_size); - weight *= (1 + (q->tc_ref / avg_top_size)); + if (likely(afl->schedule < RARE)) { + + double t = q->exec_us / avg_exec_us; + if (likely(t < 0.1)) { + + // nothing + + } else if (likely(t <= 0.25)) + + weight *= 0.9; + else if (likely(t <= 0.5)) { + + // nothing + + } else if (likely(t < 1.0)) + + weight *= 1.15; + else if (unlikely(t > 2.5 && t < 5.0)) + weight *= 1.1; + // else nothing + + } + + double l = q->len / avg_len; + if (likely(l < 0.1)) + weight *= 0.75; + else if (likely(l < 0.25)) + weight *= 1.1; + else if (unlikely(l >= 10)) + weight *= 1.1; + + double bms = q->bitmap_size / avg_bitmap_size; + if (likely(bms < 0.5)) + weight *= (1.0 + ((bms - 0.5) / 2)); + else if (unlikely(bms > 1.33)) + weight *= 1.1; - if (unlikely(weight < 0.1)) { weight = 0.1; } - if (unlikely(q->favored)) { weight *= 5; } - if (unlikely(!q->was_fuzzed)) { weight *= 2; } - if (unlikely(q->fs_redundant)) { weight *= 0.8; } + if (unlikely(!q->was_fuzzed)) { weight *= 2.5; } + if (unlikely(q->fs_redundant)) { weight *= 0.75; } return weight; @@ -117,7 +148,7 @@ void create_alias_table(afl_state_t *afl) { double avg_exec_us = 0.0; double avg_bitmap_size = 0.0; - double avg_top_size = 0.0; + double avg_len = 0.0; u32 active = 0; for (i = 0; i < n; i++) { @@ -129,7 +160,7 @@ void create_alias_table(afl_state_t *afl) { avg_exec_us += q->exec_us; avg_bitmap_size += log(q->bitmap_size); - avg_top_size += q->tc_ref; + avg_len += q->len; ++active; } @@ -138,7 +169,7 @@ void create_alias_table(afl_state_t *afl) { avg_exec_us /= active; avg_bitmap_size /= active; - avg_top_size /= active; + avg_len /= active; for (i = 0; i < n; i++) { @@ -147,7 +178,7 @@ void create_alias_table(afl_state_t *afl) { if (likely(!q->disabled)) { q->weight = - compute_weight(afl, q, avg_exec_us, avg_bitmap_size, avg_top_size); + compute_weight(afl, q, avg_exec_us, avg_bitmap_size, avg_len); q->perf_score = calculate_score(afl, q); sum += q->weight; -- cgit 1.4.1 From 477063e9ee88da5feb73b38b27a1241e8b77e002 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 6 Jun 2024 17:52:21 +0200 Subject: memory adjustments --- TODO.md | 1 + docs/Changelog.md | 1 + src/afl-fuzz-queue.c | 112 +++++++++++++++++++++++------------------------- src/afl-fuzz-redqueen.c | 34 ++++++++------- src/afl-fuzz-skipdet.c | 9 ++-- 5 files changed, 78 insertions(+), 79 deletions(-) (limited to 'docs/Changelog.md') diff --git a/TODO.md b/TODO.md index ace07434..000c78dd 100644 --- a/TODO.md +++ b/TODO.md @@ -2,6 +2,7 @@ ## Must + - review: queue_testcase_store_mem and queue_testcase_get - hardened_usercopy=0 page_alloc.shuffle=0 - add value_profile but only enable after 15 minutes without finds - cmplog max items env? diff --git a/docs/Changelog.md b/docs/Changelog.md index 633e7071..cf5d2500 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -23,6 +23,7 @@ - -V timing is now accurately the fuzz time (without syncing), before long calibration times and syncing could result in now fuzzing being made when the time was already run out until then, thanks to @eqv! + - make afl-fuzz use less memory with cmplog and fix a memleak * afl-cc: - re-enable i386 support that was accidently disabled - fixes for LTO and outdated afl-gcc mode for i386 diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index d19dd51a..cbdfd5b0 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -60,63 +60,6 @@ inline u32 select_next_queue_entry(afl_state_t *afl) { } -inline double compute_weight(afl_state_t *afl, struct queue_entry *q, - double avg_exec_us, double avg_bitmap_size, - double avg_len) { - - double weight = 1.0; - - if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) { - - u32 hits = afl->n_fuzz[q->n_fuzz_entry]; - if (likely(hits)) { weight /= (log10(hits) + 1); } - - } - - if (likely(afl->schedule < RARE)) { - - double t = q->exec_us / avg_exec_us; - if (likely(t < 0.1)) { - - // nothing - - } else if (likely(t <= 0.25)) - - weight *= 0.9; - else if (likely(t <= 0.5)) { - - // nothing - - } else if (likely(t < 1.0)) - - weight *= 1.15; - else if (unlikely(t > 2.5 && t < 5.0)) - weight *= 1.1; - // else nothing - - } - - double l = q->len / avg_len; - if (likely(l < 0.1)) - weight *= 0.75; - else if (likely(l < 0.25)) - weight *= 1.1; - else if (unlikely(l >= 10)) - weight *= 1.1; - - double bms = q->bitmap_size / avg_bitmap_size; - if (likely(bms < 0.5)) - weight *= (1.0 + ((bms - 0.5) / 2)); - else if (unlikely(bms > 1.33)) - weight *= 1.1; - - if (unlikely(!q->was_fuzzed)) { weight *= 2.5; } - if (unlikely(q->fs_redundant)) { weight *= 0.75; } - - return weight; - -} - /* create the alias table that allows weighted random selection - expensive */ void create_alias_table(afl_state_t *afl) { @@ -177,8 +120,59 @@ void create_alias_table(afl_state_t *afl) { if (likely(!q->disabled)) { - q->weight = - compute_weight(afl, q, avg_exec_us, avg_bitmap_size, avg_len); + double weight = 1.0; + { // inline does result in a compile error with LTO, weird + + if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) { + + u32 hits = afl->n_fuzz[q->n_fuzz_entry]; + if (likely(hits)) { weight /= (log10(hits) + 1); } + + } + + if (likely(afl->schedule < RARE)) { + + double t = q->exec_us / avg_exec_us; + if (likely(t < 0.1)) { + + // nothing + + } else if (likely(t <= 0.25)) + + weight *= 0.9; + else if (likely(t <= 0.5)) { + + // nothing + + } else if (likely(t < 1.0)) + + weight *= 1.15; + else if (unlikely(t > 2.5 && t < 5.0)) + weight *= 1.1; + // else nothing + + } + + double l = q->len / avg_len; + if (likely(l < 0.1)) + weight *= 0.75; + else if (likely(l < 0.25)) + weight *= 1.1; + else if (unlikely(l >= 10)) + weight *= 1.1; + + double bms = q->bitmap_size / avg_bitmap_size; + if (likely(bms < 0.5)) + weight *= (1.0 + ((bms - 0.5) / 2)); + else if (unlikely(bms > 1.33)) + weight *= 1.1; + + if (unlikely(!q->was_fuzzed)) { weight *= 2.5; } + if (unlikely(q->fs_redundant)) { weight *= 0.75; } + + } + + q->weight = weight; q->perf_score = calculate_score(afl, q); sum += q->weight; diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 9316da71..6c3582f2 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -322,7 +322,7 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, memcpy(backup, buf, len); memcpy(changed, buf, len); - if (afl->cmplog_random_colorization) { + if (likely(afl->cmplog_random_colorization)) { random_replace(afl, changed, len); @@ -402,6 +402,7 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, u32 i = 1; u32 positions = 0; + while (i) { restart: @@ -2996,15 +2997,16 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { struct tainted *t = taint; +#ifdef _DEBUG while (t) { -#ifdef _DEBUG fprintf(stderr, "T: idx=%u len=%u\n", t->pos, t->len); -#endif t = t->next; } +#endif + #if defined(_DEBUG) || defined(CMPLOG_INTROSPECTION) u64 start_time = get_cur_time(); u32 cmp_locations = 0; @@ -3148,27 +3150,27 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { exit_its: - if (afl->cmplog_lvl == CMPLOG_LVL_MAX) { + // if (afl->cmplog_lvl == CMPLOG_LVL_MAX) { - afl->queue_cur->colorized = CMPLOG_LVL_MAX; + afl->queue_cur->colorized = CMPLOG_LVL_MAX; - if (afl->queue_cur->cmplog_colorinput) { + if (afl->queue_cur->cmplog_colorinput) { - ck_free(afl->queue_cur->cmplog_colorinput); + ck_free(afl->queue_cur->cmplog_colorinput); - } + } - while (taint) { + while (taint) { - t = taint->next; - ck_free(taint); - taint = t; + t = taint->next; + ck_free(taint); + taint = t; - } + } - afl->queue_cur->taint = NULL; + afl->queue_cur->taint = NULL; - } else { + /*} else { afl->queue_cur->colorized = LVL2; @@ -3182,7 +3184,7 @@ exit_its: } - } + }*/ #ifdef CMPLOG_COMBINE if (afl->queued_items + afl->saved_crashes > orig_hit_cnt + 1) { diff --git a/src/afl-fuzz-skipdet.c b/src/afl-fuzz-skipdet.c index e52d59a3..8a927292 100644 --- a/src/afl-fuzz-skipdet.c +++ b/src/afl-fuzz-skipdet.c @@ -33,15 +33,15 @@ u8 is_det_timeout(u64 cur_ms, u8 is_flip) { u8 should_det_fuzz(afl_state_t *afl, struct queue_entry *q) { - if (!afl->skipdet_g->virgin_det_bits) { + if (unlikely(!afl->skipdet_g->virgin_det_bits)) { afl->skipdet_g->virgin_det_bits = (u8 *)ck_alloc(sizeof(u8) * afl->fsrv.map_size); } - if (!q->favored || q->passed_det) return 0; - if (!q->trace_mini) return 0; + if (likely(!q->favored || q->passed_det)) return 0; + if (unlikely(!q->trace_mini)) return 0; if (!afl->skipdet_g->last_cov_undet) afl->skipdet_g->last_cov_undet = get_cur_time(); @@ -122,7 +122,8 @@ u8 skip_deterministic_stage(afl_state_t *afl, u8 *orig_buf, u8 *out_buf, afl->stage_cur = 0; orig_hit_cnt = afl->queued_items + afl->saved_crashes; - u8 *inf_eff_map = (u8 *)ck_alloc(sizeof(u8) * len); + static u8 *inf_eff_map; + inf_eff_map = (u8 *)ck_realloc(inf_eff_map, sizeof(u8) * len); memset(inf_eff_map, 1, sizeof(u8) * len); if (common_fuzz_stuff(afl, orig_buf, len)) { return 0; } -- cgit 1.4.1 From 4bb4d4ad0060a16b08bb29533863e71f45bc3c97 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 9 Jun 2024 12:16:32 +0200 Subject: fix -n --- docs/Changelog.md | 1 + src/afl-fuzz-state.c | 3 ++- src/afl-fuzz.c | 15 ++++++++------- 3 files changed, 11 insertions(+), 8 deletions(-) (limited to 'docs/Changelog.md') diff --git a/docs/Changelog.md b/docs/Changelog.md index cf5d2500..0f4b2d8a 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -23,6 +23,7 @@ - -V timing is now accurately the fuzz time (without syncing), before long calibration times and syncing could result in now fuzzing being made when the time was already run out until then, thanks to @eqv! + - fix -n uninstrumented mode when ending fuzzing - make afl-fuzz use less memory with cmplog and fix a memleak * afl-cc: - re-enable i386 support that was accidently disabled diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index a1c1e30c..fbe6d32a 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -769,8 +769,9 @@ void afl_states_stop(void) { if (el->fsrv.fsrv_pid > 0) { kill(el->fsrv.fsrv_pid, el->fsrv.fsrv_kill_signal); + usleep(100); /* Make sure the forkserver does not end up as zombie. */ - waitpid(el->fsrv.fsrv_pid, NULL, 0); + waitpid(el->fsrv.fsrv_pid, NULL, WNOHANG); } diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 9ebe0c76..fefab1c0 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1469,15 +1469,16 @@ int main(int argc, char **argv_orig, char **envp) { #endif - configure_afl_kill_signals(&afl->fsrv, afl->afl_env.afl_child_kill_signal, - afl->afl_env.afl_fsrv_kill_signal, - (afl->fsrv.qemu_mode || afl->unicorn_mode + configure_afl_kill_signals( + &afl->fsrv, afl->afl_env.afl_child_kill_signal, + afl->afl_env.afl_fsrv_kill_signal, + (afl->fsrv.qemu_mode || afl->unicorn_mode || afl->non_instrumented_mode #ifdef __linux__ - || afl->fsrv.nyx_mode + || afl->fsrv.nyx_mode #endif - ) - ? SIGKILL - : SIGTERM); + ) + ? SIGKILL + : SIGTERM); setup_signal_handlers(); check_asan_opts(afl); -- cgit 1.4.1 From ec0b83f127702fe23da72f4d424bc13a5bacfae9 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 9 Jun 2024 18:39:56 +0200 Subject: 4.21c --- README.md | 4 ++-- TODO.md | 2 +- docs/Changelog.md | 4 ++-- include/config.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) (limited to 'docs/Changelog.md') diff --git a/README.md b/README.md index 34d73890..1b255a2a 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ AFL++ logo -Release version: [4.20c](https://github.com/AFLplusplus/AFLplusplus/releases) +Release version: [4.21c](https://github.com/AFLplusplus/AFLplusplus/releases) -GitHub version: 4.21a +GitHub version: 4.21c Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) diff --git a/TODO.md b/TODO.md index aba3cf81..b36269b4 100644 --- a/TODO.md +++ b/TODO.md @@ -3,6 +3,7 @@ ## Must - fast restart of afl-fuzz if cmdline + target hash is the same + - check for null ptr for xml/curl/g_ string transform functions - hardened_usercopy=0 page_alloc.shuffle=0 - add value_profile but only enable after 15 minutes without finds - cmplog max items env? @@ -12,7 +13,6 @@ - afl-showmap -f support - afl-fuzz multicore wrapper script - when trimming then perform crash detection - - cyclomatic complexity: 2 + calls + edges - blocks ## Should diff --git a/docs/Changelog.md b/docs/Changelog.md index 0f4b2d8a..50494acc 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -3,7 +3,7 @@ This is the list of all noteworthy changes made in every public release of the tool. See README.md for the general instruction manual. -### Version ++4.21a (dev) +### Version ++4.21c (release) * afl-fuzz - fixed a regression in afl-fuzz that resulted in a 5-10% performace loss do a switch from gettimeofday() to clock_gettime() which should be rather @@ -24,6 +24,7 @@ long calibration times and syncing could result in now fuzzing being made when the time was already run out until then, thanks to @eqv! - fix -n uninstrumented mode when ending fuzzing + - enhanced the ASAN configuration - make afl-fuzz use less memory with cmplog and fix a memleak * afl-cc: - re-enable i386 support that was accidently disabled @@ -40,7 +41,6 @@ - minor fix to collect coverage -C (thanks to @bet4it) * Fixed a shmem mmap bug (that rarely came up on MacOS) * libtokencap: script generate_libtoken_dict.sh added by @a-shvedov - * enhanced the ASAN configuration ### Version ++4.20c (release) diff --git a/include/config.h b/include/config.h index ebe40022..c4acf8db 100644 --- a/include/config.h +++ b/include/config.h @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.21a" +#define VERSION "++4.21c" /****************************************************** * * -- cgit 1.4.1