From c2b04bdf6c596f5d220f27caead20d09452ed42d Mon Sep 17 00:00:00 2001 From: Andrea Fioraldi Date: Thu, 16 Jul 2020 14:32:41 +0200 Subject: queue buffer and new splice havoc mutation --- include/afl-fuzz.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/afl-fuzz.h') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index c9f84c61..adab8155 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -546,6 +546,10 @@ typedef struct afl_state { *queue_top, /* Top of the list */ *q_prev100; /* Previous 100 marker */ + // growing buf + struct queue_entry **queue_buf; + size_t queue_size; + struct queue_entry **top_rated; /* Top entries for bitmap bytes */ struct extra_data *extras; /* Extra tokens to fuzz with */ -- cgit 1.4.1 From ce9b4698fec5222e0af1b62d68c4105e6364771e Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 21 Jul 2020 20:53:51 +0200 Subject: added andrea's splicing, added cycle_schedules --- GNUmakefile | 2 +- include/afl-fuzz.h | 11 +- include/config.h | 24 ++ include/envs.h | 1 + src/afl-fuzz-one.c | 719 +++++++++++++++++++++++++++++++++++++++++++-------- src/afl-fuzz-queue.c | 112 +++++++- src/afl-fuzz-state.c | 7 + src/afl-fuzz.c | 80 +++++- 8 files changed, 834 insertions(+), 122 deletions(-) (limited to 'include/afl-fuzz.h') diff --git a/GNUmakefile b/GNUmakefile index f44ef95e..86d6a947 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -455,7 +455,7 @@ code-format: ./.custom-format.py -i llvm_mode/*.h ./.custom-format.py -i llvm_mode/*.cc ./.custom-format.py -i gcc_plugin/*.c - #./.custom-format.py -i gcc_plugin/*.h + @#./.custom-format.py -i gcc_plugin/*.h ./.custom-format.py -i gcc_plugin/*.cc ./.custom-format.py -i custom_mutators/*/*.c ./.custom-format.py -i custom_mutators/*/*.h diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index adab8155..96d3d9f4 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -139,7 +139,8 @@ struct queue_entry { var_behavior, /* Variable behavior? */ favored, /* Currently favored? */ fs_redundant, /* Marked as redundant in the fs? */ - fully_colorized; /* Do not run redqueen stage again */ + fully_colorized, /* Do not run redqueen stage again */ + is_ascii; /* Is the input just ascii text? */ u32 bitmap_size, /* Number of bits set in bitmap */ fuzz_level; /* Number of fuzzing iterations */ @@ -333,7 +334,7 @@ typedef struct afl_env_vars { afl_dumb_forksrv, afl_import_first, afl_custom_mutator_only, afl_no_ui, afl_force_ui, afl_i_dont_care_about_missing_crashes, afl_bench_just_one, afl_bench_until_crash, afl_debug_child_output, afl_autoresume, - afl_cal_fast; + afl_cal_fast, afl_cycle_schedules; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_skip_crashes, *afl_preload; @@ -454,7 +455,9 @@ typedef struct afl_state { fixed_seed, /* do not reseed */ fast_cal, /* Try to calibrate faster? */ disable_trim, /* Never trim in fuzz_one */ - shmem_testcase_mode; /* If sharedmem testcases are used */ + shmem_testcase_mode, /* If sharedmem testcases are used */ + expand_havoc, /* perform expensive havoc after no find */ + cycle_schedules; /* cycle power schedules ? */ u8 *virgin_bits, /* Regions yet untouched by fuzzing */ *virgin_tmout, /* Bits we haven't seen in tmouts */ @@ -548,7 +551,7 @@ typedef struct afl_state { // growing buf struct queue_entry **queue_buf; - size_t queue_size; + size_t queue_size; struct queue_entry **top_rated; /* Top entries for bitmap bytes */ diff --git a/include/config.h b/include/config.h index 4503c3e9..9710cd1f 100644 --- a/include/config.h +++ b/include/config.h @@ -401,5 +401,29 @@ // #define IGNORE_FINDS +/* Text mutations */ + +/* What is the minimum length of a queue input to be evaluated for "is_ascii"? ++ */ + +#define AFL_TXT_MIN_LEN 12 + +/* What is the minimum percentage of ascii characters present to be classifed + as "is_ascii"? */ + +#define AFL_TXT_MIN_PERCENT 95 + +/* How often to perform ASCII mutations 0 = disable, 1-8 are good values */ + +#define AFL_TXT_BIAS 6 + +/* Maximum length of a string to tamper with */ + +#define AFL_TXT_STRING_MAX_LEN 1024 + +/* Maximum mutations on a string */ + +#define AFL_TXT_STRING_MAX_MUTATIONS 8 + #endif /* ! _HAVE_CONFIG_H */ diff --git a/include/envs.h b/include/envs.h index 86222418..cb3c183e 100644 --- a/include/envs.h +++ b/include/envs.h @@ -34,6 +34,7 @@ static char *afl_environment_variables[] = { "AFL_CUSTOM_MUTATOR_LIBRARY", "AFL_CUSTOM_MUTATOR_ONLY", "AFL_CXX", + "AFL_CYCLE_SCHEDULES", "AFL_DEBUG", "AFL_DEBUG_CHILD_OUTPUT", "AFL_DEBUG_GDB", diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 399bfcab..250409da 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -24,6 +24,11 @@ */ #include "afl-fuzz.h" +#include + +static u8 *strnstr(const u8 *s, const u8 *find, size_t slen); +static u32 string_replace(u8 **out_buf, s32 *temp_len, u32 pos, u8 *from, + u8 *to); /* MOpt */ @@ -362,6 +367,450 @@ static void locate_diffs(u8 *ptr1, u8 *ptr2, u32 len, s32 *first, s32 *last) { #endif /* !IGNORE_FINDS */ +#define BUF_PARAMS(name) (void **)&afl->name##_buf, &afl->name##_size + +/* search a string */ + +static u8 *strnstr(const u8 *s, const u8 *find, size_t slen) { + + char c, sc; + size_t len; + + if ((c = *find++) != '\0') { + + len = strlen(find); + do { + + do { + + if (slen-- < 1 || (sc = *s++) == '\0') return (NULL); + + } while (sc != c); + + if (len > slen) return (NULL); + + } while (strncmp(s, find, len) != 0); + + s--; + + } + + return ((u8 *)s); + +} + +/* replace between deliminators, if rep == NULL, then we will duplicate the + * target */ + +static u32 delim_replace(u8 **out_buf, s32 *temp_len, size_t pos, + const u8 *ldelim, const u8 *rdelim, u8 *rep) { + + u8 *end_buf = *out_buf + *temp_len; + u8 *ldelim_start = strnstr(*out_buf + pos, ldelim, *temp_len - pos); + + if (ldelim_start != NULL) { + + u32 max = (end_buf - ldelim_start - 1 > AFL_TXT_STRING_MAX_LEN + ? AFL_TXT_STRING_MAX_LEN + : end_buf - ldelim_start - 1); + + if (max > 0) { + + u8 *rdelim_end = strnstr(ldelim_start + 1, rdelim, max); + + if (rdelim_end != NULL) { + + u32 rep_len, delim_space_len = rdelim_end - ldelim_start - 1, xtra = 0; + + if (rep != NULL) { + + rep_len = (u32)strlen(rep); + + } else { // NULL? then we copy the value in between the delimiters + + rep_len = delim_space_len; + delim_space_len = 0; + rep = ldelim_start + 1; + xtra = rep_len; + + } + + if (rep_len != delim_space_len) { + + memmove(ldelim_start + rep_len + xtra + 1, rdelim_end, + *temp_len - (rdelim_end - *out_buf)); + + } + + memcpy(ldelim_start + 1, rep, rep_len); + *temp_len = (*temp_len - delim_space_len + rep_len); + + return 1; + + } + + } + + } + + return 0; + +} + +static u32 delim_swap(u8 **out_buf, s32 *temp_len, size_t pos, const u8 *ldelim, + const u8 *mdelim, const u8 *rdelim) { + + u8 *out_buf_end = *out_buf + *temp_len; + u32 max = (*temp_len - pos > AFL_TXT_STRING_MAX_LEN ? AFL_TXT_STRING_MAX_LEN + : *temp_len - pos); + u8 *ldelim_start = strnstr(*out_buf + pos, ldelim, max); + + if (ldelim_start != NULL) { + + max = (out_buf_end - ldelim_start - 1 > AFL_TXT_STRING_MAX_LEN + ? AFL_TXT_STRING_MAX_LEN + : out_buf_end - ldelim_start - 1); + if (max > 1) { + + u8 *mdelim_pos = strnstr(ldelim_start + 1, mdelim, max); + + if (mdelim_pos != NULL) { + + max = (out_buf_end - mdelim_pos - 1 > AFL_TXT_STRING_MAX_LEN + ? AFL_TXT_STRING_MAX_LEN + : out_buf_end - mdelim_pos - 1); + if (max > 0) { + + u8 *rdelim_end = strnstr(mdelim + 1, rdelim, max); + + if (rdelim_end != NULL) { + + u32 first_len = mdelim_pos - ldelim_start - 1; + u32 second_len = rdelim_end - mdelim_pos - 1; + u8 scratch[AFL_TXT_STRING_MAX_LEN]; + + memcpy(scratch, ldelim_start + 1, first_len); + + if (first_len != second_len) { + + memmove(ldelim_start + second_len + 1, mdelim_pos, + out_buf_end - mdelim_pos); + + } + + memcpy(ldelim_start + 1, mdelim_pos + 1, second_len); + + if (first_len != second_len) { + + memmove(mdelim_pos + first_len + 1, rdelim_end, + out_buf_end - rdelim_end); + + } + + memcpy(mdelim_pos + 1, scratch, first_len); + + return 1; + + } + + } + + } + + } + + } + + return 0; + +} + +/* replace a string */ + +static u32 string_replace(u8 **out_buf, s32 *temp_len, u32 pos, u8 *from, + u8 *to) { + + u8 *start = strnstr(*out_buf + pos, from, *temp_len - pos); + + if (start) { + + u32 from_len = strlen(from); + u32 to_len = strlen(to); + + if (from_len != to_len) { + + memmove(start + to_len, start + from_len, + *temp_len - from_len - (start - *out_buf)); + + } + + memcpy(start, to, to_len); + *temp_len = (*temp_len - from_len + to_len); + + return 1; + + } + + return 0; + +} + +/* Returns 1 if a mutant was generated and placed in out_buf, 0 if none + * generated. */ + +static int text_mutation(afl_state_t *afl, u8 **out_buf, s32 *orig_temp_len) { + + s32 temp_len; + u32 pos, yes = 0, + mutations = rand_below(afl, AFL_TXT_STRING_MAX_MUTATIONS) + 1; + u8 *new_buf = ck_maybe_grow(BUF_PARAMS(out_scratch), + *orig_temp_len + AFL_TXT_STRING_MAX_MUTATIONS); + temp_len = *orig_temp_len; + memcpy(new_buf, *out_buf, temp_len); + + for (u32 i = 0; i < mutations; i++) { + + if (temp_len < AFL_TXT_MIN_LEN) { return 0; } + + pos = rand_below(afl, temp_len - 1); + int choice = rand_below(afl, 72); + switch (choice) { + + case 0: /* Semantic statement deletion */ + yes += string_replace(out_buf, &temp_len, pos, "\n", "#"); + break; + case 1: + yes += string_replace(out_buf, &temp_len, pos, "(", "(!"); + break; + case 2: + yes += string_replace(out_buf, &temp_len, pos, "==", "!="); + break; + case 3: + yes += string_replace(out_buf, &temp_len, pos, "!=", "=="); + break; + case 4: + yes += string_replace(out_buf, &temp_len, pos, "==", "<"); + break; + case 5: + yes += string_replace(out_buf, &temp_len, pos, "<", "=="); + break; + case 6: + yes += string_replace(out_buf, &temp_len, pos, "==", ">"); + break; + case 7: + yes += string_replace(out_buf, &temp_len, pos, ">", "=="); + break; + case 8: + yes += string_replace(out_buf, &temp_len, pos, "=", "<"); + break; + case 9: + yes += string_replace(out_buf, &temp_len, pos, "=", ">"); + break; + case 10: + yes += string_replace(out_buf, &temp_len, pos, "<", ">"); + break; + case 11: + yes += string_replace(out_buf, &temp_len, pos, ">", "<"); + break; + case 12: + yes += string_replace(out_buf, &temp_len, pos, "++", "--"); + break; + case 13: + yes += string_replace(out_buf, &temp_len, pos, "--", "++"); + break; + case 14: + yes += string_replace(out_buf, &temp_len, pos, "+", "-"); + break; + case 15: + yes += string_replace(out_buf, &temp_len, pos, "+", "*"); + break; + case 16: + yes += string_replace(out_buf, &temp_len, pos, "+", "/"); + break; + case 17: + yes += string_replace(out_buf, &temp_len, pos, "+", "%"); + break; + case 18: + yes += string_replace(out_buf, &temp_len, pos, "*", "-"); + break; + case 19: + yes += string_replace(out_buf, &temp_len, pos, "*", "+"); + break; + case 20: + yes += string_replace(out_buf, &temp_len, pos, "*", "/"); + break; + case 21: + yes += string_replace(out_buf, &temp_len, pos, "*", "%"); + break; + case 22: + yes += string_replace(out_buf, &temp_len, pos, "-", "+"); + break; + case 23: + yes += string_replace(out_buf, &temp_len, pos, "-", "*"); + break; + case 24: + yes += string_replace(out_buf, &temp_len, pos, "-", "/"); + break; + case 25: + yes += string_replace(out_buf, &temp_len, pos, "-", "%"); + break; + case 26: + yes += string_replace(out_buf, &temp_len, pos, "/", "-"); + break; + case 27: + yes += string_replace(out_buf, &temp_len, pos, "/", "*"); + break; + case 28: + yes += string_replace(out_buf, &temp_len, pos, "/", "+"); + break; + case 29: + yes += string_replace(out_buf, &temp_len, pos, "/", "%"); + break; + case 30: + yes += string_replace(out_buf, &temp_len, pos, "%", "-"); + break; + case 31: + yes += string_replace(out_buf, &temp_len, pos, "%", "*"); + break; + case 32: + yes += string_replace(out_buf, &temp_len, pos, "%", "/"); + break; + case 33: + yes += string_replace(out_buf, &temp_len, pos, "%", "+"); + break; + case 34: + yes += string_replace(out_buf, &temp_len, pos, " ", "|"); + break; + case 35: + yes += string_replace(out_buf, &temp_len, pos, " ", "$"); + break; + case 36: + yes += string_replace(out_buf, &temp_len, pos, "0", "1"); + break; + case 37: + yes += string_replace(out_buf, &temp_len, pos, "1", "0"); + break; + case 38: + yes += string_replace(out_buf, &temp_len, pos, " ", "`"); + break; + case 39: + yes += string_replace(out_buf, &temp_len, pos, " ", "\""); + break; + case 40: + yes += string_replace(out_buf, &temp_len, pos, ";", " "); + break; + case 41: + yes += string_replace(out_buf, &temp_len, pos, "&&", "||"); + break; + case 42: + yes += string_replace(out_buf, &temp_len, pos, "||", "&&"); + break; + case 43: + yes += string_replace(out_buf, &temp_len, pos, "!", ""); + break; + case 44: + yes += string_replace(out_buf, &temp_len, pos, "==", "="); + break; + case 45: + yes += string_replace(out_buf, &temp_len, pos, "--", ""); + break; + case 46: + yes += string_replace(out_buf, &temp_len, pos, "<<", "<"); + break; + case 47: + yes += string_replace(out_buf, &temp_len, pos, ">>", ">"); + break; + case 48: + yes += string_replace(out_buf, &temp_len, pos, "<", "<<"); + break; + case 49: + yes += string_replace(out_buf, &temp_len, pos, ">", ">>"); + break; + case 50: + yes += string_replace(out_buf, &temp_len, pos, "\"", "'"); + break; + case 51: + yes += string_replace(out_buf, &temp_len, pos, "'", "\""); + break; + case 52: + yes += string_replace(out_buf, &temp_len, pos, "(", "\""); + break; + case 53: /* Remove a semicolon delimited statement after a semicolon */ + yes += delim_replace(out_buf, &temp_len, pos, ";", ";", ";"); + break; + case 54: /* Remove a semicolon delimited statement after a left curly + brace */ + yes += delim_replace(out_buf, &temp_len, pos, "}", ";", "}"); + break; + case 55: /* Remove a curly brace construct */ + yes += delim_replace(out_buf, &temp_len, pos, "{", "}", ""); + break; + case 56: /* Replace a curly brace construct with an empty one */ + yes += delim_replace(out_buf, &temp_len, pos, "{", "}", "{}"); + break; + case 57: + yes += delim_swap(out_buf, &temp_len, pos, ";", ";", ";"); + break; + case 58: + yes += delim_swap(out_buf, &temp_len, pos, "}", ";", ";"); + break; + case 59: /* Swap comma delimited things case 1 */ + yes += delim_swap(out_buf, &temp_len, pos, "(", ",", ")"); + break; + case 60: /* Swap comma delimited things case 2 */ + yes += delim_swap(out_buf, &temp_len, pos, "(", ",", ","); + break; + case 61: /* Swap comma delimited things case 3 */ + yes += delim_swap(out_buf, &temp_len, pos, ",", ",", ","); + break; + case 62: /* Swap comma delimited things case 4 */ + yes += delim_swap(out_buf, &temp_len, pos, ",", ",", ")"); + break; + case 63: /* Just delete a line */ + yes += delim_replace(out_buf, &temp_len, pos, "\n", "\n", ""); + break; + case 64: /* Delete something like "const" case 1 */ + yes += delim_replace(out_buf, &temp_len, pos, " ", " ", ""); + break; + case 65: /* Delete something like "const" case 2 */ + yes += delim_replace(out_buf, &temp_len, pos, "\n", " ", ""); + break; + case 66: /* Delete something like "const" case 3 */ + yes += delim_replace(out_buf, &temp_len, pos, "(", " ", ""); + break; + case 67: /* Swap space delimited things case 1 */ + yes += delim_swap(out_buf, &temp_len, pos, " ", " ", " "); + break; + case 68: /* Swap space delimited things case 2 */ + yes += delim_swap(out_buf, &temp_len, pos, " ", " ", ")"); + break; + case 69: /* Swap space delimited things case 3 */ + yes += delim_swap(out_buf, &temp_len, pos, "(", " ", " "); + break; + case 70: /* Swap space delimited things case 4 */ + yes += delim_swap(out_buf, &temp_len, pos, "(", " ", ")"); + break; + case 71: /* Duplicate a single line of code */ + yes += delim_replace(out_buf, &temp_len, pos, "\n", "\n", NULL); + break; + case 72: /* Duplicate a construct (most often, a non-nested for loop */ + yes += delim_replace(out_buf, &temp_len, pos, "\n", "}", NULL); + break; + + } + + } + + if (yes == 0 || temp_len <= 0) { return 0; } + + swap_bufs(BUF_PARAMS(out), BUF_PARAMS(out_scratch)); + *out_buf = new_buf; + *orig_temp_len = temp_len; + + return 1; + +} + /* Take the current entry from the queue, fuzz it for a while. This function is a tad too long... returns 0 if fuzzed successfully, 1 if skipped or bailed out. */ @@ -1854,6 +2303,22 @@ havoc_stage: /* We essentially just do several thousand runs (depending on perf_score) where we take the input file and make random stacked tweaks. */ + u32 r_max, r; + + if (unlikely(afl->expand_havoc)) { + + /* add expensive havoc cases here, they are activated after a full + cycle without finds happened */ + + r_max = 16 + ((afl->extras_cnt + afl->a_extras_cnt) ? 2 : 0) + + (afl->queue_cur->is_ascii ? AFL_TXT_BIAS : 0); + + } else { + + r_max = 15 + ((afl->extras_cnt + afl->a_extras_cnt) ? 2 : 0); + + } + for (afl->stage_cur = 0; afl->stage_cur < afl->stage_max; ++afl->stage_cur) { u32 use_stacking = 1 << (1 + rand_below(afl, HAVOC_STACK_POW2)); @@ -1896,8 +2361,9 @@ havoc_stage: } - switch (rand_below( - afl, 16 + ((afl->extras_cnt + afl->a_extras_cnt) ? 2 : 0))) { + retry_havoc: + + switch ((r = rand_below(afl, r_max))) { case 0: @@ -2191,176 +2657,203 @@ havoc_stage: break; } - - case 15: { - /* Overwrite bytes with a randomly selected chunk from another - testcase or insert that chunk. */ + default: - if (afl->queued_paths < 2) break; + if (likely(r <= 16 && (afl->extras_cnt || afl->a_extras_cnt))) { - /* Pick a random queue entry and seek to it. */ + /* Values 15 and 16 can be selected only if there are any extras + present in the dictionaries. */ - u32 tid; - do - tid = rand_below(afl, afl->queued_paths); - while (tid == afl->current_entry); + if (r == 15) { - struct queue_entry* target = afl->queue_buf[tid]; + /* Overwrite bytes with an extra. */ - /* Make sure that the target has a reasonable length. */ + if (!afl->extras_cnt || + (afl->a_extras_cnt && rand_below(afl, 2))) { - while (target && (target->len < 2 || target == afl->queue_cur)) - target = target->next; + /* No user-specified extras or odds in our favor. Let's use an + auto-detected one. */ - if (!target) break; + u32 use_extra = rand_below(afl, afl->a_extras_cnt); + u32 extra_len = afl->a_extras[use_extra].len; + u32 insert_at; - /* Read the testcase into a new buffer. */ + if (extra_len > temp_len) { break; } - fd = open(target->fname, O_RDONLY); + insert_at = rand_below(afl, temp_len - extra_len + 1); + memcpy(out_buf + insert_at, afl->a_extras[use_extra].data, + extra_len); - if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", target->fname); } + } else { - u32 new_len = target->len; - u8 * new_buf = ck_maybe_grow(BUF_PARAMS(in_scratch), new_len); + /* No auto extras or odds in our favor. Use the dictionary. */ - ck_read(fd, new_buf, new_len, target->fname); + u32 use_extra = rand_below(afl, afl->extras_cnt); + u32 extra_len = afl->extras[use_extra].len; + u32 insert_at; - close(fd); + if (extra_len > temp_len) { break; } - u8 overwrite = 0; - if (temp_len >= 2 && rand_below(afl, 2)) - overwrite = 1; - else if (temp_len + HAVOC_BLK_XL >= MAX_FILE) { - if (temp_len >= 2) overwrite = 1; - else break; - } + insert_at = rand_below(afl, temp_len - extra_len + 1); + memcpy(out_buf + insert_at, afl->extras[use_extra].data, + extra_len); - if (overwrite) { + } - u32 copy_from, copy_to, copy_len; + break; - copy_len = choose_block_len(afl, new_len - 1); - if (copy_len > temp_len) copy_len = temp_len; + } else { // case 16 - copy_from = rand_below(afl, new_len - copy_len + 1); - copy_to = rand_below(afl, temp_len - copy_len + 1); + u32 use_extra, extra_len, + insert_at = rand_below(afl, temp_len + 1); + u8 *ptr; - memmove(out_buf + copy_to, new_buf + copy_from, copy_len); + /* Insert an extra. Do the same dice-rolling stuff as for the + previous case. */ - } else { - - u32 clone_from, clone_to, clone_len; - - clone_len = choose_block_len(afl, new_len); - clone_from = rand_below(afl, new_len - clone_len + 1); - - clone_to = rand_below(afl, temp_len); + if (!afl->extras_cnt || + (afl->a_extras_cnt && rand_below(afl, 2))) { - u8 * temp_buf = - ck_maybe_grow(BUF_PARAMS(out_scratch), temp_len + clone_len); + use_extra = rand_below(afl, afl->a_extras_cnt); + extra_len = afl->a_extras[use_extra].len; + ptr = afl->a_extras[use_extra].data; - /* Head */ + } else { - memcpy(temp_buf, out_buf, clone_to); + use_extra = rand_below(afl, afl->extras_cnt); + extra_len = afl->extras[use_extra].len; + ptr = afl->extras[use_extra].data; - /* Inserted part */ + } - memcpy(temp_buf + clone_to, new_buf + clone_from, clone_len); + if (temp_len + extra_len >= MAX_FILE) { break; } - /* Tail */ - memcpy(temp_buf + clone_to + clone_len, out_buf + clone_to, - temp_len - clone_to); + out_buf = ck_maybe_grow(BUF_PARAMS(out), temp_len + extra_len); - swap_bufs(BUF_PARAMS(out), BUF_PARAMS(out_scratch)); - out_buf = temp_buf; - temp_len += clone_len; - - } + /* Tail */ + memmove(out_buf + insert_at + extra_len, out_buf + insert_at, + temp_len - insert_at); - break; + /* Inserted part */ + memcpy(out_buf + insert_at, ptr, extra_len); - } + temp_len += extra_len; - /* Values 15 and 16 can be selected only if there are any extras - present in the dictionaries. */ + break; - case 16: { + } - /* Overwrite bytes with an extra. */ + } else - if (!afl->extras_cnt || (afl->a_extras_cnt && rand_below(afl, 2))) { + switch (r) { - /* No user-specified extras or odds in our favor. Let's use an - auto-detected one. */ + case 15: // fall through + case 17: { - u32 use_extra = rand_below(afl, afl->a_extras_cnt); - u32 extra_len = afl->a_extras[use_extra].len; - u32 insert_at; + /* Overwrite bytes with a randomly selected chunk from another + testcase or insert that chunk. */ - if (extra_len > temp_len) { break; } + if (afl->queued_paths < 2) break; - insert_at = rand_below(afl, temp_len - extra_len + 1); - memcpy(out_buf + insert_at, afl->a_extras[use_extra].data, - extra_len); + /* Pick a random queue entry and seek to it. */ - } else { + u32 tid; + do + tid = rand_below(afl, afl->queued_paths); + while (tid == afl->current_entry); - /* No auto extras or odds in our favor. Use the dictionary. */ + struct queue_entry *target = afl->queue_buf[tid]; - u32 use_extra = rand_below(afl, afl->extras_cnt); - u32 extra_len = afl->extras[use_extra].len; - u32 insert_at; + /* Make sure that the target has a reasonable length. */ - if (extra_len > temp_len) { break; } + while (target && (target->len < 2 || target == afl->queue_cur)) + target = target->next; - insert_at = rand_below(afl, temp_len - extra_len + 1); - memcpy(out_buf + insert_at, afl->extras[use_extra].data, extra_len); + if (!target) break; - } + /* Read the testcase into a new buffer. */ - break; + fd = open(target->fname, O_RDONLY); - } + if (unlikely(fd < 0)) { + + PFATAL("Unable to open '%s'", target->fname); - case 17: { + } - u32 use_extra, extra_len, insert_at = rand_below(afl, temp_len + 1); - u8 *ptr; + u32 new_len = target->len; + u8 *new_buf = ck_maybe_grow(BUF_PARAMS(in_scratch), new_len); - /* Insert an extra. Do the same dice-rolling stuff as for the - previous case. */ + ck_read(fd, new_buf, new_len, target->fname); - if (!afl->extras_cnt || (afl->a_extras_cnt && rand_below(afl, 2))) { + close(fd); - use_extra = rand_below(afl, afl->a_extras_cnt); - extra_len = afl->a_extras[use_extra].len; - ptr = afl->a_extras[use_extra].data; + u8 overwrite = 0; + if (temp_len >= 2 && rand_below(afl, 2)) + overwrite = 1; + else if (temp_len + HAVOC_BLK_XL >= MAX_FILE) { - } else { + if (temp_len >= 2) + overwrite = 1; + else + break; - use_extra = rand_below(afl, afl->extras_cnt); - extra_len = afl->extras[use_extra].len; - ptr = afl->extras[use_extra].data; + } - } + if (overwrite) { - if (temp_len + extra_len >= MAX_FILE) { break; } + u32 copy_from, copy_to, copy_len; - out_buf = ck_maybe_grow(BUF_PARAMS(out), temp_len + extra_len); + copy_len = choose_block_len(afl, new_len - 1); + if (copy_len > temp_len) copy_len = temp_len; - /* Tail */ - memmove(out_buf + insert_at + extra_len, out_buf + insert_at, - temp_len - insert_at); + copy_from = rand_below(afl, new_len - copy_len + 1); + copy_to = rand_below(afl, temp_len - copy_len + 1); - /* Inserted part */ - memcpy(out_buf + insert_at, ptr, extra_len); + memmove(out_buf + copy_to, new_buf + copy_from, copy_len); - temp_len += extra_len; + } else { - break; + u32 clone_from, clone_to, clone_len; - } + clone_len = choose_block_len(afl, new_len); + clone_from = rand_below(afl, new_len - clone_len + 1); + + clone_to = rand_below(afl, temp_len); + + u8 *temp_buf = ck_maybe_grow(BUF_PARAMS(out_scratch), + temp_len + clone_len); + + /* Head */ + + memcpy(temp_buf, out_buf, clone_to); + + /* Inserted part */ + + memcpy(temp_buf + clone_to, new_buf + clone_from, clone_len); + + /* Tail */ + memcpy(temp_buf + clone_to + clone_len, out_buf + clone_to, + temp_len - clone_to); + + swap_bufs(BUF_PARAMS(out), BUF_PARAMS(out_scratch)); + out_buf = temp_buf; + temp_len += clone_len; + + } + + break; + + } + + default: + + // perform ascii mutations + if (text_mutation(afl, &out_buf, &temp_len) == 0) + goto retry_havoc; + + } // end default: switch(r) } @@ -4827,7 +5320,7 @@ u8 fuzz_one(afl_state_t *afl) { return (key_val_lv_1 | key_val_lv_2); -#undef BUF_PARAMS - } +#undef BUF_PARAMS + diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index a96995e5..56073b0a 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -24,6 +24,7 @@ #include "afl-fuzz.h" #include +#include #define BUF_PARAMS(name) (void **)&afl->name##_buf, &afl->name##_size @@ -102,6 +103,108 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) { } +/* check if ascii or UTF-8 */ + +static u8 check_if_text(struct queue_entry *q) { + + if (q->len < AFL_TXT_MIN_LEN) return 0; + + u8 buf[MAX_FILE]; + s32 fd, len = q->len, offset = 0, ascii = 0, utf8 = 0, comp; + + if ((fd = open(q->fname, O_RDONLY)) < 0) return 0; + if ((comp = read(fd, buf, len)) != len) return 0; + close(fd); + + while (offset < len) { + + // ASCII: <= 0x7F to allow ASCII control characters + if ((buf[offset + 0] == 0x09 || buf[offset + 0] == 0x0A || + buf[offset + 0] == 0x0D || + (0x20 <= buf[offset + 0] && buf[offset + 0] <= 0x7E))) { + + offset++; + utf8++; + ascii++; + continue; + + } + + if (isascii((int)buf[offset]) || isprint((int)buf[offset])) { + + ascii++; + // we continue though as it can also be a valid utf8 + + } + + // non-overlong 2-byte + if (((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF))) { + + offset += 2; + utf8++; + comp--; + continue; + + } + + // excluding overlongs + if ((buf[offset + 0] == 0xE0 && + (0xA0 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && + buf[offset + 2] <= 0xBF)) || // straight 3-byte + (((0xE1 <= buf[offset + 0] && buf[offset + 0] <= 0xEC) || + buf[offset + 0] == 0xEE || buf[offset + 0] == 0xEF) && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && + buf[offset + 2] <= 0xBF)) || // excluding surrogates + (buf[offset + 0] == 0xED && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0x9F) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF))) { + + offset += 3; + utf8++; + comp -= 2; + continue; + + } + + // planes 1-3 + if ((buf[offset + 0] == 0xF0 && + (0x90 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) && + (0x80 <= buf[offset + 3] && + buf[offset + 3] <= 0xBF)) || // planes 4-15 + ((0xF1 <= buf[offset + 0] && buf[offset + 0] <= 0xF3) && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) && + (0x80 <= buf[offset + 3] && buf[offset + 3] <= 0xBF)) || // plane 16 + (buf[offset + 0] == 0xF4 && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0x8F) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) && + (0x80 <= buf[offset + 3] && buf[offset + 3] <= 0xBF))) { + + offset += 4; + utf8++; + comp -= 3; + continue; + + } + + offset++; + + } + + u32 percent_utf8 = (utf8 * 100) / comp; + u32 percent_ascii = (ascii * 100) / len; + + if (percent_utf8 >= percent_ascii && percent_utf8 >= AFL_TXT_MIN_PERCENT) + return 2; + if (percent_ascii >= AFL_TXT_MIN_PERCENT) return 1; + return 0; + +} + /* Append new test case to the queue. */ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { @@ -139,9 +242,10 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { afl->q_prev100 = q; } - - struct queue_entry** queue_buf = ck_maybe_grow(BUF_PARAMS(queue), afl->queued_paths * sizeof(struct queue_entry*)); - queue_buf[afl->queued_paths -1] = q; + + struct queue_entry **queue_buf = ck_maybe_grow( + BUF_PARAMS(queue), afl->queued_paths * sizeof(struct queue_entry *)); + queue_buf[afl->queued_paths - 1] = q; afl->last_path_time = get_cur_time(); @@ -164,6 +268,8 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { } + q->is_ascii = check_if_text(q); + } /* Destroy the entire queue. */ diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index e56d122a..f68a79e8 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -293,6 +293,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_autoresume = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_CYCLE_SCHEDULES", + + afl_environment_variable_len)) { + + afl->cycle_schedules = afl->afl_env.afl_cycle_schedules = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_CAL_FAST", afl_environment_variable_len)) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index df2896d2..88f8e902 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1252,11 +1252,42 @@ int main(int argc, char **argv_orig, char **envp) { /* If we had a full queue cycle with no new finds, try recombination strategies next. */ - if (afl->queued_paths == prev_queued) { + if (afl->queued_paths == prev_queued && + (get_cur_time() - afl->start_time) >= 3600) { if (afl->use_splicing) { ++afl->cycles_wo_finds; + switch (afl->expand_havoc) { + + case 0: + afl->expand_havoc = 1; + break; + case 1: + if (afl->limit_time_sig == 0) { + + afl->limit_time_sig = -1; + afl->limit_time_puppet = 0; + + } + + afl->expand_havoc = 2; + break; + case 2: + afl->cycle_schedules = 1; + afl->expand_havoc = 3; + break; + case 3: + // nothing else currently + break; + + } + + if (afl->expand_havoc) { + + } else + + afl->expand_havoc = 1; } else { @@ -1270,6 +1301,53 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->cycle_schedules) { + + /* we cannot mix non-AFLfast schedules with others */ + + switch (afl->schedule) { + + case EXPLORE: + afl->schedule = EXPLOIT; + break; + case EXPLOIT: + afl->schedule = MMOPT; + break; + case MMOPT: + afl->schedule = SEEK; + break; + case SEEK: + afl->schedule = EXPLORE; + break; + case FAST: + afl->schedule = COE; + break; + case COE: + afl->schedule = LIN; + break; + case LIN: + afl->schedule = QUAD; + break; + case QUAD: + afl->schedule = RARE; + break; + case RARE: + afl->schedule = FAST; + break; + + } + + struct queue_entry *q = afl->queue; + // we must recalculate the scores of all queue entries + while (q) { + + update_bitmap_score(afl, q); + q = q->next; + + } + + } + prev_queued = afl->queued_paths; if (afl->sync_id && afl->queue_cycle == 1 && -- cgit 1.4.1 From 9cddbc04206bd8d1399e5a5311c98fff5be80731 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 24 Jul 2020 12:26:52 +0200 Subject: add -F option to sync to foreign fuzzer queues --- GNUmakefile | 4 +- README.md | 20 +++--- TODO.md | 2 - docs/Changelog.md | 2 + docs/parallel_fuzzing.md | 14 ++++- include/afl-fuzz.h | 13 ++++ src/afl-fuzz-init.c | 154 +++++++++++++++++++++++++++++++++++++++++++++++ src/afl-fuzz-run.c | 2 + src/afl-fuzz.c | 22 ++++++- 9 files changed, 211 insertions(+), 22 deletions(-) (limited to 'include/afl-fuzz.h') diff --git a/GNUmakefile b/GNUmakefile index f44ef95e..ab9144b8 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -455,10 +455,10 @@ code-format: ./.custom-format.py -i llvm_mode/*.h ./.custom-format.py -i llvm_mode/*.cc ./.custom-format.py -i gcc_plugin/*.c - #./.custom-format.py -i gcc_plugin/*.h + @#./.custom-format.py -i gcc_plugin/*.h ./.custom-format.py -i gcc_plugin/*.cc ./.custom-format.py -i custom_mutators/*/*.c - ./.custom-format.py -i custom_mutators/*/*.h + @#./.custom-format.py -i custom_mutators/*/*.h # destroys input.h :-( ./.custom-format.py -i examples/*/*.c ./.custom-format.py -i examples/*/*.h ./.custom-format.py -i test/*.c diff --git a/README.md b/README.md index 4e83021d..b2f41315 100644 --- a/README.md +++ b/README.md @@ -366,9 +366,9 @@ If you find other good ones, please send them to us :-) ## Power schedules -The power schedules were copied from Marcel Böhme's excellent AFLfast -implementation and expand on the ability to discover new paths and -therefore may increase the code coverage. +The power schedules were copied from Marcel Böhme's AFLfast implementation and +measure differently which queue entries to prefer and therefore may find +different paths faster for large queues. The available schedules are: @@ -382,16 +382,10 @@ The available schedules are: - mmopt (afl++ experimental) - seek (afl++ experimental) -In parallel mode (-M/-S, several instances with the shared queue), we suggest to -run the main node using the explore or fast schedule (-p explore) and the secondary -nodes with a combination of cut-off-exponential (-p coe), exponential (-p fast), -explore (-p explore) and mmopt (-p mmopt) schedules. If a schedule does -not perform well for a target, restart the secondary nodes with a different schedule. - -In single mode, using -p fast is usually slightly more beneficial than the -default explore mode. -(We don't want to change the default behavior of afl, so "fast" has not been -made the default mode). +In parallel mode (-M/-S, several instances with the shared queue), we suggest +to run the main node using the default explore schedule (`-p explore`) and the +secondary nodes with different schedules. If a schedule does not perform well +for a target, restart the secondary nodes with a different schedule. More details can be found in the paper published at the 23rd ACM Conference on Computer and Communications Security [CCS'16](https://www.sigsac.org/ccs/CCS2016/accepted-papers/) diff --git a/TODO.md b/TODO.md index ad3ef83e..ad743b6b 100644 --- a/TODO.md +++ b/TODO.md @@ -3,9 +3,7 @@ ## Roadmap 2.67+ - -i - + foreign fuzzer sync support: scandir with time sort - - pre_save custom module example to save away test cases - expand on AFL_LLVM_INSTRUMENT_FILE to also support sancov allowlist format - - allow to sync against honggfuzz and libfuzzer - AFL_MAP_SIZE for qemu_mode and unicorn_mode - namespace for targets? e.g. network - learn from honggfuzz (mutations, maybe ptrace?) diff --git a/docs/Changelog.md b/docs/Changelog.md index a25cc43c..bec87d65 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -11,6 +11,8 @@ sending a mail to . ### Version ++2.66d (devel) - afl-fuzz: + - added -F option to allow -M main fuzzers to sync to foreign fuzzers, + e.g. honggfuzz or libfuzzer - eliminated CPU affinity race condition for -S/-M runs - llvm_mode: - fixes for laf-intel float splitting (thanks to mark-griffin for diff --git a/docs/parallel_fuzzing.md b/docs/parallel_fuzzing.md index 271f8369..2ab1466c 100644 --- a/docs/parallel_fuzzing.md +++ b/docs/parallel_fuzzing.md @@ -99,7 +99,15 @@ example may be: This is not a concern if you use @@ without -f and let afl-fuzz come up with the file name. -## 3) Multi-system parallelization +## 3) Syncing with non-afl fuzzers or independant instances + +A -M main node can be told with the `-F other_fuzzer_queue_directory` option +to sync results from other fuzzers, e.g. libfuzzer or honggfuzz. + +Only the specified directory will by synced into afl, not subdirectories. +The specified directories do not need to exist yet at the start of afl. + +## 4) Multi-system parallelization The basic operating principle for multi-system parallelization is similar to the mechanism explained in section 2. The key difference is that you need to @@ -176,7 +184,7 @@ It is *not* advisable to skip the synchronization script and run the fuzzers directly on a network filesystem; unexpected latency and unkillable processes in I/O wait state can mess things up. -## 4) Remote monitoring and data collection +## 5) Remote monitoring and data collection You can use screen, nohup, tmux, or something equivalent to run remote instances of afl-fuzz. If you redirect the program's output to a file, it will @@ -200,7 +208,7 @@ Keep in mind that crashing inputs are *not* automatically propagated to the main instance, so you may still want to monitor for crashes fleet-wide from within your synchronization or health checking scripts (see afl-whatsup). -## 5) Asymmetric setups +## 6) Asymmetric setups It is perhaps worth noting that all of the following is permitted: diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index c9f84c61..cf4254ac 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -347,6 +347,13 @@ struct afl_pass_stat { }; +struct foreign_sync { + + u8 * dir; + time_t ctime; + +}; + typedef struct afl_state { /* Position of this state in the global states list */ @@ -574,6 +581,11 @@ typedef struct afl_state { u8 describe_op_buf_256[256]; /* describe_op will use this to return a string up to 256 */ +/* foreign sync */ +#define FOREIGN_SYNCS_MAX 32 + u8 foreign_sync_cnt; + struct foreign_sync foreign_syncs[FOREIGN_SYNCS_MAX]; + #ifdef _AFL_DOCUMENT_MUTATIONS u8 do_document; u32 document_counter; @@ -937,6 +949,7 @@ void fix_up_banner(afl_state_t *, u8 *); void check_if_tty(afl_state_t *); void setup_signal_handlers(void); void save_cmdline(afl_state_t *, u32, char **); +void read_foreign_testcases(afl_state_t *, int); /* CmpLog */ diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 609e16ba..65ad0c9f 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -438,6 +438,159 @@ static void shuffle_ptrs(afl_state_t *afl, void **ptrs, u32 cnt) { } +/* Read all testcases from foreign input directories, then queue them for + testing. Called at startup and at sync intervals. + Does not descend into subdirectories! */ + +void read_foreign_testcases(afl_state_t *afl, int first) { + + if (!afl->foreign_sync_cnt) return; + + struct dirent **nl; + s32 nl_cnt; + u32 i, iter; + + u8 val_buf[2][STRINGIFY_VAL_SIZE_MAX]; + + for (iter = 0; iter < afl->foreign_sync_cnt; iter++) { + + if (afl->foreign_syncs[iter].dir != NULL && + afl->foreign_syncs[iter].dir[0] != 0) { + + if (first) ACTF("Scanning '%s'...", afl->foreign_syncs[iter].dir); + time_t ctime_max = 0; + + /* We use scandir() + alphasort() rather than readdir() because otherwise, + the ordering of test cases would vary somewhat randomly and would be + difficult to control. */ + + nl_cnt = scandir(afl->foreign_syncs[iter].dir, &nl, NULL, NULL); + + if (nl_cnt < 0) { + + if (first) { + + WARNF("Unable to open directory '%s'", afl->foreign_syncs[iter].dir); + sleep(1); + + } + + continue; + + } + + if (nl_cnt == 0) { + + if (first) + WARNF("directory %s is currently empty", + afl->foreign_syncs[iter].dir); + continue; + + } + + /* Show stats */ + + snprintf(afl->stage_name_buf, STAGE_BUF_SIZE, "foreign sync %u", iter); + + afl->stage_name = afl->stage_name_buf; + afl->stage_cur = 0; + afl->stage_max = 0; + + for (i = 0; i < nl_cnt; ++i) { + + struct stat st; + + u8 *fn2 = + alloc_printf("%s/%s", afl->foreign_syncs[iter].dir, nl[i]->d_name); + + free(nl[i]); /* not tracked */ + + if (unlikely(lstat(fn2, &st) || access(fn2, R_OK))) { + + if (first) PFATAL("Unable to access '%s'", fn2); + continue; + + } + + /* we detect new files by their ctime */ + if (likely(st.st_ctime <= afl->foreign_syncs[iter].ctime)) { + + ck_free(fn2); + continue; + + } + + /* This also takes care of . and .. */ + + if (!S_ISREG(st.st_mode) || !st.st_size || strstr(fn2, "/README.txt")) { + + ck_free(fn2); + continue; + + } + + if (st.st_size > MAX_FILE) { + + if (first) + WARNF( + "Test case '%s' is too big (%s, limit is %s), skipping", fn2, + stringify_mem_size(val_buf[0], sizeof(val_buf[0]), st.st_size), + stringify_mem_size(val_buf[1], sizeof(val_buf[1]), MAX_FILE)); + ck_free(fn2); + continue; + + } + + // lets do not use add_to_queue(afl, fn2, st.st_size, 0); + // as this could add duplicates of the startup input corpus + + int fd = open(fn2, O_RDONLY); + if (fd < 0) { + + ck_free(fn2); + continue; + + } + + u8 fault; + u8 *mem = mmap(0, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + + if (mem == MAP_FAILED) { + + ck_free(fn2); + continue; + + } + + write_to_testcase(afl, mem, st.st_size); + fault = fuzz_run_target(afl, &afl->fsrv, afl->fsrv.exec_tmout); + afl->syncing_party = "foreign"; + afl->queued_imported += + save_if_interesting(afl, mem, st.st_size, fault); + afl->syncing_party = 0; + munmap(mem, st.st_size); + close(fd); + + if (st.st_ctime > ctime_max) ctime_max = st.st_ctime; + + } + + afl->foreign_syncs[iter].ctime = ctime_max; + free(nl); /* not tracked */ + + } + + } + + if (first) { + + afl->last_path_time = 0; + afl->queued_at_start = afl->queued_paths; + + } + +} + /* Read all testcases from the input directory, then queue them for testing. Called at startup. */ @@ -530,6 +683,7 @@ void read_testcases(afl_state_t *afl) { WARNF("Test case '%s' is too big (%s, limit is %s), skipping", fn2, stringify_mem_size(val_buf[0], sizeof(val_buf[0]), st.st_size), stringify_mem_size(val_buf[1], sizeof(val_buf[1]), MAX_FILE)); + ck_free(fn2); continue; } diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 2a1664e2..6e3be72b 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -612,6 +612,8 @@ void sync_fuzzers(afl_state_t *afl) { } + if (afl->foreign_sync_cnt) read_foreign_testcases(afl, 0); + } /* Trim all new test cases to save cycles when doing deterministic checks. The diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index df2896d2..f03c545d 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -131,10 +131,13 @@ static void usage(afl_state_t *afl, u8 *argv0, int more_help) { "executions.\n\n" "Other stuff:\n" - " -T text - text banner to show on the screen\n" " -M/-S id - distributed mode (see docs/parallel_fuzzing.md)\n" " use -D to force -S secondary to perform deterministic " "fuzzing\n" + " -F path - sync to a foreign fuzzer queue directory (requires " + "-M, can\n" + " be specified up to %u times)\n" + " -T text - text banner to show on the screen\n" " -I command - execute this command/script when a new crash is " "found\n" //" -B bitmap.txt - mutate a specific test case, use the out/fuzz_bitmap @@ -142,7 +145,7 @@ static void usage(afl_state_t *afl, u8 *argv0, int more_help) { " -C - crash exploration mode (the peruvian rabbit thing)\n" " -e ext - file extension for the fuzz test input file (if " "needed)\n\n", - argv0, EXEC_TIMEOUT, MEM_LIMIT); + argv0, EXEC_TIMEOUT, MEM_LIMIT, FOREIGN_SYNCS_MAX); if (more_help > 1) { @@ -403,6 +406,19 @@ int main(int argc, char **argv_orig, char **envp) { afl->use_splicing = 1; break; + case 'F': /* foreign sync dir */ + + if (!afl->is_main_node) + FATAL( + "Option -F can only be specified after the -M option for the " + "main fuzzer of a fuzzing campaign"); + if (afl->foreign_sync_cnt >= FOREIGN_SYNCS_MAX) + FATAL("Maximum %u entried of -F option can be specified", + FOREIGN_SYNCS_MAX); + afl->foreign_syncs[afl->foreign_sync_cnt].dir = optarg; + afl->foreign_sync_cnt++; + break; + case 'f': /* target file */ if (afl->fsrv.out_file) { FATAL("Multiple -f options not supported"); } @@ -1059,6 +1075,8 @@ int main(int argc, char **argv_orig, char **envp) { setup_cmdline_file(afl, argv + optind); read_testcases(afl); + // read_foreign_testcases(afl, 1); for the moment dont do this + load_auto(afl); pivot_inputs(afl); -- cgit 1.4.1 From 30c09915432af7a9e98f9b4d8b09566731e0cca9 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 24 Jul 2020 13:26:07 +0200 Subject: better text mutation --- include/afl-fuzz.h | 2 +- include/config.h | 7 +- src/afl-fuzz-one.c | 341 +++++++++++++++++++++++++----------------------- src/afl-fuzz-redqueen.c | 142 ++++++++++---------- src/afl-fuzz-state.c | 7 + src/afl-fuzz.c | 2 +- 6 files changed, 261 insertions(+), 240 deletions(-) (limited to 'include/afl-fuzz.h') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 96d3d9f4..0e2b7458 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -334,7 +334,7 @@ typedef struct afl_env_vars { afl_dumb_forksrv, afl_import_first, afl_custom_mutator_only, afl_no_ui, afl_force_ui, afl_i_dont_care_about_missing_crashes, afl_bench_just_one, afl_bench_until_crash, afl_debug_child_output, afl_autoresume, - afl_cal_fast, afl_cycle_schedules; + afl_cal_fast, afl_cycle_schedules, afl_expand_havoc; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_skip_crashes, *afl_preload; diff --git a/include/config.h b/include/config.h index 9710cd1f..344a368f 100644 --- a/include/config.h +++ b/include/config.h @@ -403,15 +403,14 @@ /* Text mutations */ -/* What is the minimum length of a queue input to be evaluated for "is_ascii"? -+ */ +/* Minimum length of a queue input to be evaluated for "is_ascii"? */ #define AFL_TXT_MIN_LEN 12 /* What is the minimum percentage of ascii characters present to be classifed as "is_ascii"? */ -#define AFL_TXT_MIN_PERCENT 95 +#define AFL_TXT_MIN_PERCENT 94 /* How often to perform ASCII mutations 0 = disable, 1-8 are good values */ @@ -423,7 +422,7 @@ /* Maximum mutations on a string */ -#define AFL_TXT_STRING_MAX_MUTATIONS 8 +#define AFL_TXT_STRING_MAX_MUTATIONS 6 #endif /* ! _HAVE_CONFIG_H */ diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 7f96c2c6..dc19150d 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -559,13 +559,25 @@ static u32 string_replace(u8 **out_buf, s32 *temp_len, u32 pos, u8 *from, /* Returns 1 if a mutant was generated and placed in out_buf, 0 if none * generated. */ +static const uint8_t text_mutation_special_chars[] = { + + '\t', '\n', '\r', ' ', '!', '"', '$', '%', '&', '\'', '(', ')', '*', + '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', + '\\', ']', '^', '_', '`', '{', '|', '}', '~', ' ' // space is here twice + +}; + static int text_mutation(afl_state_t *afl, u8 **out_buf, s32 *orig_temp_len) { + if (*orig_temp_len < AFL_TXT_MIN_LEN) { return 0; } + s32 temp_len; u32 pos, yes = 0, - mutations = rand_below(afl, AFL_TXT_STRING_MAX_MUTATIONS) + 1; - u8 *new_buf = ck_maybe_grow(BUF_PARAMS(out_scratch), - *orig_temp_len + AFL_TXT_STRING_MAX_MUTATIONS +1); + mutations = rand_below(afl, AFL_TXT_STRING_MAX_MUTATIONS) + 16; + u8 *new_buf = + ck_maybe_grow(BUF_PARAMS(out_scratch), + *orig_temp_len + AFL_TXT_STRING_MAX_MUTATIONS + 16); + u8 fromc[2] = {0, 0}, toc[2] = {0, 0}; temp_len = *orig_temp_len; memcpy(new_buf, *out_buf, temp_len); new_buf[temp_len] = 0; @@ -575,9 +587,12 @@ static int text_mutation(afl_state_t *afl, u8 **out_buf, s32 *orig_temp_len) { if (temp_len < AFL_TXT_MIN_LEN) { return 0; } pos = rand_below(afl, temp_len - 1); - int choice = rand_below(afl, 80); + int choice = rand_below(afl, 100); + switch (choice) { + /* 50% -> fixed replacements */ + case 0: /* Semantic statement deletion */ yes += string_replace(&new_buf, &temp_len, pos, "\n", "#"); break; @@ -624,246 +639,240 @@ static int text_mutation(afl_state_t *afl, u8 **out_buf, s32 *orig_temp_len) { yes += string_replace(&new_buf, &temp_len, pos, "+", "-"); break; case 15: - yes += string_replace(&new_buf, &temp_len, pos, "+", "*"); + yes += string_replace(&new_buf, &temp_len, pos, "-", "+"); break; case 16: - yes += string_replace(&new_buf, &temp_len, pos, "+", "/"); + yes += string_replace(&new_buf, &temp_len, pos, "0", "1"); break; case 17: - yes += string_replace(&new_buf, &temp_len, pos, "+", "%"); + yes += string_replace(&new_buf, &temp_len, pos, "1", "0"); break; case 18: - yes += string_replace(&new_buf, &temp_len, pos, "*", "-"); + yes += string_replace(&new_buf, &temp_len, pos, "&&", "||"); break; case 19: - yes += string_replace(&new_buf, &temp_len, pos, "*", "+"); + yes += string_replace(&new_buf, &temp_len, pos, "||", "&&"); break; case 20: - yes += string_replace(&new_buf, &temp_len, pos, "*", "/"); + yes += string_replace(&new_buf, &temp_len, pos, "!", ""); break; case 21: - yes += string_replace(&new_buf, &temp_len, pos, "*", "%"); + yes += string_replace(&new_buf, &temp_len, pos, "==", "="); break; case 22: - yes += string_replace(&new_buf, &temp_len, pos, "-", "+"); + yes += string_replace(&new_buf, &temp_len, pos, "=", "=="); break; case 23: - yes += string_replace(&new_buf, &temp_len, pos, "-", "*"); + yes += string_replace(&new_buf, &temp_len, pos, "--", ""); break; case 24: - yes += string_replace(&new_buf, &temp_len, pos, "-", "/"); + yes += string_replace(&new_buf, &temp_len, pos, "<<", "<"); break; case 25: - yes += string_replace(&new_buf, &temp_len, pos, "-", "%"); + yes += string_replace(&new_buf, &temp_len, pos, ">>", ">"); break; case 26: - yes += string_replace(&new_buf, &temp_len, pos, "/", "-"); + yes += string_replace(&new_buf, &temp_len, pos, "<", "<<"); break; case 27: - yes += string_replace(&new_buf, &temp_len, pos, "/", "*"); + yes += string_replace(&new_buf, &temp_len, pos, ">", ">>"); break; case 28: - yes += string_replace(&new_buf, &temp_len, pos, "/", "+"); + yes += string_replace(&new_buf, &temp_len, pos, "'", "\""); break; case 29: - yes += string_replace(&new_buf, &temp_len, pos, "/", "%"); - break; - case 30: - yes += string_replace(&new_buf, &temp_len, pos, "%", "-"); - break; - case 31: - yes += string_replace(&new_buf, &temp_len, pos, "%", "*"); - break; - case 32: - yes += string_replace(&new_buf, &temp_len, pos, "%", "/"); - break; - case 33: - yes += string_replace(&new_buf, &temp_len, pos, "%", "+"); - break; - case 34: - yes += string_replace(&new_buf, &temp_len, pos, " ", "|"); - break; - case 35: - yes += string_replace(&new_buf, &temp_len, pos, " ", "$"); - break; - case 36: - yes += string_replace(&new_buf, &temp_len, pos, "0", "1"); - break; - case 37: - yes += string_replace(&new_buf, &temp_len, pos, "1", "0"); - break; - case 38: - yes += string_replace(&new_buf, &temp_len, pos, " ", "`"); - break; - case 39: - yes += string_replace(&new_buf, &temp_len, pos, " ", "\""); - break; - case 40: - yes += string_replace(&new_buf, &temp_len, pos, ";", " "); - break; - case 41: - yes += string_replace(&new_buf, &temp_len, pos, "&&", "||"); - break; - case 42: - yes += string_replace(&new_buf, &temp_len, pos, "||", "&&"); - break; - case 43: - yes += string_replace(&new_buf, &temp_len, pos, "!", ""); - break; - case 44: - yes += string_replace(&new_buf, &temp_len, pos, "==", "="); - break; - case 45: - yes += string_replace(&new_buf, &temp_len, pos, "--", ""); - break; - case 46: - yes += string_replace(&new_buf, &temp_len, pos, "<<", "<"); - break; - case 47: - yes += string_replace(&new_buf, &temp_len, pos, ">>", ">"); - break; - case 48: - yes += string_replace(&new_buf, &temp_len, pos, "<", "<<"); - break; - case 49: - yes += string_replace(&new_buf, &temp_len, pos, ">", ">>"); - break; - case 50: yes += string_replace(&new_buf, &temp_len, pos, "\"", "'"); break; - case 51: - yes += string_replace(&new_buf, &temp_len, pos, "'", "\""); - break; - case 52: - yes += string_replace(&new_buf, &temp_len, pos, "(", "\""); - break; - case 53: /* Remove a semicolon delimited statement after a semicolon */ + case 30: /* Remove a semicolon delimited statement after a semicolon */ yes += delim_replace(&new_buf, &temp_len, pos, ";", ";", ";"); break; - case 54: /* Remove a semicolon delimited statement after a left curly + case 31: /* Remove a semicolon delimited statement after a left curly brace */ yes += delim_replace(&new_buf, &temp_len, pos, "}", ";", "}"); break; - case 55: /* Remove a curly brace construct */ + case 32: /* Remove a curly brace construct */ yes += delim_replace(&new_buf, &temp_len, pos, "{", "}", ""); break; - case 56: /* Replace a curly brace construct with an empty one */ + case 33: /* Replace a curly brace construct with an empty one */ yes += delim_replace(&new_buf, &temp_len, pos, "{", "}", "{}"); break; - case 57: + case 34: yes += delim_swap(&new_buf, &temp_len, pos, ";", ";", ";"); break; - case 58: + case 35: yes += delim_swap(&new_buf, &temp_len, pos, "}", ";", ";"); break; - case 59: /* Swap comma delimited things case 1 */ + case 36: /* Swap comma delimited things case 1 */ yes += delim_swap(&new_buf, &temp_len, pos, "(", ",", ")"); break; - case 60: /* Swap comma delimited things case 2 */ + case 37: /* Swap comma delimited things case 2 */ yes += delim_swap(&new_buf, &temp_len, pos, "(", ",", ","); break; - case 61: /* Swap comma delimited things case 3 */ + case 38: /* Swap comma delimited things case 3 */ yes += delim_swap(&new_buf, &temp_len, pos, ",", ",", ","); break; - case 62: /* Swap comma delimited things case 4 */ + case 39: /* Swap comma delimited things case 4 */ yes += delim_swap(&new_buf, &temp_len, pos, ",", ",", ")"); break; - case 63: /* Just delete a line */ + case 40: /* Just delete a line */ yes += delim_replace(&new_buf, &temp_len, pos, "\n", "\n", ""); break; - case 64: /* Delete something like "const" case 1 */ + case 41: /* Delete something like "const" case 1 */ yes += delim_replace(&new_buf, &temp_len, pos, " ", " ", ""); break; - case 65: /* Delete something like "const" case 2 */ + case 42: /* Delete something like "const" case 2 */ yes += delim_replace(&new_buf, &temp_len, pos, "\n", " ", ""); break; - case 66: /* Delete something like "const" case 3 */ + case 43: /* Delete something like "const" case 3 */ yes += delim_replace(&new_buf, &temp_len, pos, "(", " ", ""); break; - case 67: /* Swap space delimited things case 1 */ + case 44: /* Swap space delimited things case 1 */ yes += delim_swap(&new_buf, &temp_len, pos, " ", " ", " "); break; - case 68: /* Swap space delimited things case 2 */ + case 45: /* Swap space delimited things case 2 */ yes += delim_swap(&new_buf, &temp_len, pos, " ", " ", ")"); break; - case 69: /* Swap space delimited things case 3 */ + case 46: /* Swap space delimited things case 3 */ yes += delim_swap(&new_buf, &temp_len, pos, "(", " ", " "); break; - case 70: /* Swap space delimited things case 4 */ + case 47: /* Swap space delimited things case 4 */ yes += delim_swap(&new_buf, &temp_len, pos, "(", " ", ")"); break; - case 71: /* Duplicate a single line of code */ + case 48: /* Duplicate a single line of code */ yes += delim_replace(&new_buf, &temp_len, pos, "\n", "\n", NULL); break; - case 72: /* Duplicate a construct (most often, a non-nested for loop */ + case 49: /* Duplicate a construct (most often, a non-nested for loop */ yes += delim_replace(&new_buf, &temp_len, pos, "\n", "}", NULL); break; default: { - - for (u32 j = pos; j < temp_len; ++j) { - if (isdigit(new_buf[j])) { - - new_buf[temp_len] = 0; // should be safe thanks to the initial grow - - u8* endptr; - unsigned long long num = strtoull(new_buf +j, (char**)&endptr, 0); - - switch (rand_below(afl, 8)) { - case 0: - num = rand_below(afl, INT_MAX); - break; - case 1: - num = rand_next(afl); - break; - case 2: - num += 1 + rand_below(afl, 255); - break; - case 3: - num -= 1 + rand_below(afl, 255); - break; - case 4: - num *= 1 + rand_below(afl, 255); - break; - case 5: - num /= 1 + rand_below(afl, 255); - break; - case 6: - num /= 1 + rand_below(afl, 255); - break; - case 7: - num = ~num; - break; - } - - const char* fmt = "%llu"; - if (rand_below(afl, 5) == 0) // add - sign with 1/5 probability - fmt = "-%llu"; - - size_t num_len = snprintf(NULL, 0, fmt, num); - size_t old_len = endptr - (new_buf +j); - if (num_len < old_len) { - memmove(new_buf +j +num_len, new_buf +j +old_len, temp_len - (j + old_len)); - snprintf(new_buf +j, num_len, fmt, num); - temp_len -= old_len - num_len; - } else if (num_len == old_len) { - snprintf(new_buf +j, num_len, fmt, num); - } else { - new_buf = ck_maybe_grow(BUF_PARAMS(out_scratch), temp_len + (num_len - old_len) + AFL_TXT_STRING_MAX_MUTATIONS +1); - memmove(new_buf +j +num_len, new_buf +j +old_len, temp_len - (j + old_len)); - snprintf(new_buf +j, num_len, fmt, num); - temp_len += num_len - old_len; + + /* 10% is transforming ascii numbers */ + + if (choice < 60) { + + for (u32 j = pos; j < temp_len; ++j) { + + if (isdigit(new_buf[j])) { + + new_buf[temp_len] = + 0; // should be safe thanks to the initial grow + + u8 * endptr; + unsigned long long num = + strtoull(new_buf + j, (char **)&endptr, 0); + + switch (rand_below(afl, 8)) { + + case 0: + num = rand_below(afl, INT_MAX); + break; + case 1: + num = rand_next(afl); + break; + case 2: + num += 1 + rand_below(afl, 255); + break; + case 3: + num -= 1 + rand_below(afl, 255); + break; + case 4: + num *= 1 + rand_below(afl, 255); + break; + case 5: + num /= 1 + rand_below(afl, 255); + break; + case 6: + num /= 1 + rand_below(afl, 255); + break; + case 7: + num = ~num; + break; + + } + + const char *fmt = "%llu"; + if (rand_below(afl, 5) == 0) // add - sign with 1/5 probability + fmt = "-%llu"; + + size_t num_len = snprintf(NULL, 0, fmt, num); + size_t old_len = endptr - (new_buf + j); + if (num_len < old_len) { + + memmove(new_buf + j + num_len, new_buf + j + old_len, + temp_len - (j + old_len)); + snprintf(new_buf + j, num_len, fmt, num); + temp_len -= old_len - num_len; + + } else if (num_len == old_len) { + + snprintf(new_buf + j, num_len, fmt, num); + + } else { + + new_buf = ck_maybe_grow(BUF_PARAMS(out_scratch), + temp_len + (num_len - old_len) + + AFL_TXT_STRING_MAX_MUTATIONS + 1); + memmove(new_buf + j + num_len, new_buf + j + old_len, + temp_len - (j + old_len)); + snprintf(new_buf + j, num_len, fmt, num); + temp_len += num_len - old_len; + + } + + yes += 1; + break; + } - yes += 1; - break; - } + + } else if (choice < 90) { + + /* 30% is special character transform */ + + fromc[0] = text_mutation_special_chars[rand_below( + afl, sizeof(text_mutation_special_chars))]; + + do { + + toc[0] = text_mutation_special_chars[rand_below( + afl, sizeof(text_mutation_special_chars))]; + + } while (toc[0] == fromc[0]); + + yes += string_replace(&new_buf, &temp_len, pos, fromc, toc); + break; + + } else { + + /* 10% is random text character transform */ + + u32 iter, cnt, loc, prev_loc = temp_len; + if (temp_len > 32) { + + cnt = 1 + rand_below(afl, 5); + + } else { + + cnt = rand_below(afl, 2); + + } + + for (iter = 0; iter <= cnt; iter++) { + + while ((loc = rand_below(afl, temp_len)) == prev_loc) + ; + new_buf[loc] = 32 + rand_below(afl, 'z' - ' ' + 1); + prev_loc = loc; + + } + } - + } } - + } if (yes == 0 || temp_len <= 0) { return 0; } @@ -871,7 +880,7 @@ static int text_mutation(afl_state_t *afl, u8 **out_buf, s32 *orig_temp_len) { swap_bufs(BUF_PARAMS(out), BUF_PARAMS(out_scratch)); *out_buf = new_buf; *orig_temp_len = temp_len; - + return 1; } diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index de3adb2d..57e60c3d 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -264,49 +264,53 @@ static u8 its_fuzz(afl_state_t *afl, u8 *buf, u32 len, u8 *status) { } static long long strntoll(const char *str, size_t sz, char **end, int base) { - char buf[64]; - long long ret; - const char *beg = str; - - for (; beg && sz && *beg == ' '; beg++, sz--) - ; - - if (!sz || sz >= sizeof(buf)) { - if (end) - *end = (char *)str; - return 0; - } - - memcpy(buf, beg, sz); - buf[sz] = '\0'; - ret = strtoll(buf, end, base); - if (ret == LLONG_MIN || ret == LLONG_MAX) - return ret; - if (end) - *end = (char *)beg + (*end - buf); - return ret; + + char buf[64]; + long long ret; + const char *beg = str; + + for (; beg && sz && *beg == ' '; beg++, sz--) + ; + + if (!sz || sz >= sizeof(buf)) { + + if (end) *end = (char *)str; + return 0; + + } + + memcpy(buf, beg, sz); + buf[sz] = '\0'; + ret = strtoll(buf, end, base); + if (ret == LLONG_MIN || ret == LLONG_MAX) return ret; + if (end) *end = (char *)beg + (*end - buf); + return ret; + } -static unsigned long long strntoull(const char *str, size_t sz, char **end, int base) { - char buf[64]; - unsigned long long ret; - const char *beg = str; - - for (; beg && sz && *beg == ' '; beg++, sz--) - ; - - if (!sz || sz >= sizeof(buf)) { - if (end) - *end = (char *)str; - return 0; - } - - memcpy(buf, beg, sz); - buf[sz] = '\0'; - ret = strtoull(buf, end, base); - if (end) - *end = (char *)beg + (*end - buf); - return ret; +static unsigned long long strntoull(const char *str, size_t sz, char **end, + int base) { + + char buf[64]; + unsigned long long ret; + const char * beg = str; + + for (; beg && sz && *beg == ' '; beg++, sz--) + ; + + if (!sz || sz >= sizeof(buf)) { + + if (end) *end = (char *)str; + return 0; + + } + + memcpy(buf, beg, sz); + buf[sz] = '\0'; + ret = strtoull(buf, end, base); + if (end) *end = (char *)beg + (*end - buf); + return ret; + } #define BUF_PARAMS(name) (void **)&afl->name##_buf, &afl->name##_size @@ -328,49 +332,51 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u32 its_len = len - idx; // *status = 0; - u8 *endptr; - u8 use_num = 0, use_unum = 0; + u8 * endptr; + u8 use_num = 0, use_unum = 0; unsigned long long unum; - long long num; + long long num; if (afl->queue_cur->is_ascii) { - + endptr = buf_8; - num = strntoll(buf_8, len - idx, (char**)&endptr, 0); + num = strntoll(buf_8, len - idx, (char **)&endptr, 0); if (endptr == buf_8) { - unum = strntoull(buf_8, len - idx, (char**)&endptr, 0); - if (endptr == buf_8) - use_unum = 1; + + unum = strntoull(buf_8, len - idx, (char **)&endptr, 0); + if (endptr == buf_8) use_unum = 1; + } else + use_num = 1; - + } - + if (use_num && num == pattern) { - + size_t old_len = endptr - buf_8; size_t num_len = snprintf(NULL, 0, "%lld", num); - - u8* new_buf = ck_maybe_grow(BUF_PARAMS(out_scratch), len + num_len); + + u8 *new_buf = ck_maybe_grow(BUF_PARAMS(out_scratch), len + num_len); memcpy(new_buf, buf, idx); - - snprintf(new_buf +idx, num_len, "%lld", num); - memcpy(new_buf +idx +num_len, buf_8 + old_len, len - idx - old_len); - + + snprintf(new_buf + idx, num_len, "%lld", num); + memcpy(new_buf + idx + num_len, buf_8 + old_len, len - idx - old_len); + if (unlikely(its_fuzz(afl, new_buf, len, status))) { return 1; } - + } else if (use_unum && unum == pattern) { - + size_t old_len = endptr - buf_8; size_t num_len = snprintf(NULL, 0, "%llu", unum); - - u8* new_buf = ck_maybe_grow(BUF_PARAMS(out_scratch), len + num_len); + + u8 *new_buf = ck_maybe_grow(BUF_PARAMS(out_scratch), len + num_len); memcpy(new_buf, buf, idx); - - snprintf(new_buf +idx, num_len, "%llu", unum); - memcpy(new_buf +idx +num_len, buf_8 + old_len, len - idx - old_len); - + + snprintf(new_buf + idx, num_len, "%llu", unum); + memcpy(new_buf + idx + num_len, buf_8 + old_len, len - idx - old_len); + if (unlikely(its_fuzz(afl, new_buf, len, status))) { return 1; } - + } if (SHAPE_BYTES(h->shape) >= 8 && *status != 1) { @@ -382,7 +388,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, *buf_64 = pattern; } - + // reverse encoding if (do_reverse && *status != 1) { diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index f68a79e8..66280ed1 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -300,6 +300,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->cycle_schedules = afl->afl_env.afl_cycle_schedules = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_EXPAND_HAVOC_NOW", + + afl_environment_variable_len)) { + + afl->expand_havoc = afl->afl_env.afl_expand_havoc = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_CAL_FAST", afl_environment_variable_len)) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 553300e9..c014035e 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1275,7 +1275,7 @@ int main(int argc, char **argv_orig, char **envp) { afl->expand_havoc = 2; break; case 2: - //afl->cycle_schedules = 1; + // afl->cycle_schedules = 1; afl->expand_havoc = 3; break; case 3: -- cgit 1.4.1 From 16e362d2b93a60d6c50fca6abfabd9976ca6142d Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sun, 26 Jul 2020 15:55:03 +0200 Subject: add last 60s exec/s stat --- include/afl-fuzz.h | 4 ++ src/afl-fuzz-mutators.c | 3 +- src/afl-fuzz-stats.c | 154 +++++++++++++++++++++++++--------------------- test/test-floatingpoint.c | 8 +-- 4 files changed, 93 insertions(+), 76 deletions(-) (limited to 'include/afl-fuzz.h') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index cf4254ac..c0c4cfd5 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -581,6 +581,10 @@ typedef struct afl_state { u8 describe_op_buf_256[256]; /* describe_op will use this to return a string up to 256 */ + unsigned long long int last_avg_exec_update; + u32 last_avg_execs; + float last_avg_execs_saved; + /* foreign sync */ #define FOREIGN_SYNCS_MAX 32 u8 foreign_sync_cnt; diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index ed777811..850266c2 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -168,7 +168,8 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { /* "afl_custom_deinit", optional for backward compatibility */ mutator->afl_custom_deinit = dlsym(dh, "afl_custom_deinit"); - if (!mutator->afl_custom_deinit) FATAL("Symbol 'afl_custom_deinit' not found."); + if (!mutator->afl_custom_deinit) + FATAL("Symbol 'afl_custom_deinit' not found."); /* "afl_custom_post_process", optional */ mutator->afl_custom_post_process = dlsym(dh, "afl_custom_post_process"); diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index fc93011b..995f298e 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -39,7 +39,7 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability, u8 fn[PATH_MAX]; s32 fd; FILE * f; - uint32_t t_bytes = count_non_255_bytes(afl, afl->virgin_bits); + u32 t_bytes = count_non_255_bytes(afl, afl->virgin_bits); snprintf(fn, PATH_MAX, "%s/fuzzer_stats", afl->out_dir); @@ -67,89 +67,101 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability, } + if ((unlikely(!afl->last_avg_exec_update || + cur_time - afl->last_avg_exec_update >= 60000))) { + + afl->last_avg_execs_saved = + (float)(1000*(afl->fsrv.total_execs - afl->last_avg_execs)) / + (float)(cur_time - afl->last_avg_exec_update); + afl->last_avg_execs = afl->fsrv.total_execs; + afl->last_avg_exec_update = cur_time; + + } + #ifndef __HAIKU__ if (getrusage(RUSAGE_CHILDREN, &rus)) { rus.ru_maxrss = 0; } #endif - fprintf( - f, - "start_time : %llu\n" - "last_update : %llu\n" - "run_time : %llu\n" - "fuzzer_pid : %u\n" - "cycles_done : %llu\n" - "cycles_wo_finds : %llu\n" - "execs_done : %llu\n" - "execs_per_sec : %0.02f\n" - // "real_execs_per_sec: %0.02f\n" // damn the name is too long - "paths_total : %u\n" - "paths_favored : %u\n" - "paths_found : %u\n" - "paths_imported : %u\n" - "max_depth : %u\n" - "cur_path : %u\n" /* Must match find_start_position() */ - "pending_favs : %u\n" - "pending_total : %u\n" - "variable_paths : %u\n" - "stability : %0.02f%%\n" - "bitmap_cvg : %0.02f%%\n" - "unique_crashes : %llu\n" - "unique_hangs : %llu\n" - "last_path : %llu\n" - "last_crash : %llu\n" - "last_hang : %llu\n" - "execs_since_crash : %llu\n" - "exec_timeout : %u\n" - "slowest_exec_ms : %u\n" - "peak_rss_mb : %lu\n" - "cpu_affinity : %d\n" - "edges_found : %u\n" - "var_byte_count : %u\n" - "afl_banner : %s\n" - "afl_version : " VERSION - "\n" - "target_mode : %s%s%s%s%s%s%s%s%s\n" - "command_line : %s\n", - afl->start_time / 1000, cur_time / 1000, - (cur_time - afl->start_time) / 1000, (u32)getpid(), - afl->queue_cycle ? (afl->queue_cycle - 1) : 0, afl->cycles_wo_finds, - afl->fsrv.total_execs, - afl->fsrv.total_execs / - ((double)(get_cur_time() - afl->start_time) / 1000), - afl->queued_paths, afl->queued_favored, afl->queued_discovered, - afl->queued_imported, afl->max_depth, afl->current_entry, - afl->pending_favored, afl->pending_not_fuzzed, afl->queued_variable, - stability, bitmap_cvg, afl->unique_crashes, afl->unique_hangs, - afl->last_path_time / 1000, afl->last_crash_time / 1000, - afl->last_hang_time / 1000, afl->fsrv.total_execs - afl->last_crash_execs, - afl->fsrv.exec_tmout, afl->slowest_exec_ms, + fprintf(f, + "start_time : %llu\n" + "last_update : %llu\n" + "run_time : %llu\n" + "fuzzer_pid : %u\n" + "cycles_done : %llu\n" + "cycles_wo_finds : %llu\n" + "execs_done : %llu\n" + "execs_per_sec : %0.02f\n" + "execs_ps_last_min : %0.02f\n" + "paths_total : %u\n" + "paths_favored : %u\n" + "paths_found : %u\n" + "paths_imported : %u\n" + "max_depth : %u\n" + "cur_path : %u\n" /* Must match find_start_position() */ + "pending_favs : %u\n" + "pending_total : %u\n" + "variable_paths : %u\n" + "stability : %0.02f%%\n" + "bitmap_cvg : %0.02f%%\n" + "unique_crashes : %llu\n" + "unique_hangs : %llu\n" + "last_path : %llu\n" + "last_crash : %llu\n" + "last_hang : %llu\n" + "execs_since_crash : %llu\n" + "exec_timeout : %u\n" + "slowest_exec_ms : %u\n" + "peak_rss_mb : %lu\n" + "cpu_affinity : %d\n" + "edges_found : %u\n" + "var_byte_count : %u\n" + "afl_banner : %s\n" + "afl_version : " VERSION + "\n" + "target_mode : %s%s%s%s%s%s%s%s%s\n" + "command_line : %s\n", + afl->start_time / 1000, cur_time / 1000, + (cur_time - afl->start_time) / 1000, (u32)getpid(), + afl->queue_cycle ? (afl->queue_cycle - 1) : 0, afl->cycles_wo_finds, + afl->fsrv.total_execs, + afl->fsrv.total_execs / + ((double)(get_cur_time() - afl->start_time) / 1000), + afl->last_avg_execs_saved, afl->queued_paths, afl->queued_favored, + afl->queued_discovered, afl->queued_imported, afl->max_depth, + afl->current_entry, afl->pending_favored, afl->pending_not_fuzzed, + afl->queued_variable, stability, bitmap_cvg, afl->unique_crashes, + afl->unique_hangs, afl->last_path_time / 1000, + afl->last_crash_time / 1000, afl->last_hang_time / 1000, + afl->fsrv.total_execs - afl->last_crash_execs, afl->fsrv.exec_tmout, + afl->slowest_exec_ms, #ifndef __HAIKU__ #ifdef __APPLE__ - (unsigned long int)(rus.ru_maxrss >> 20), + (unsigned long int)(rus.ru_maxrss >> 20), #else - (unsigned long int)(rus.ru_maxrss >> 10), + (unsigned long int)(rus.ru_maxrss >> 10), #endif #else - -1UL, + -1UL, #endif #ifdef HAVE_AFFINITY - afl->cpu_aff, + afl->cpu_aff, #else - -1, + -1, #endif - t_bytes, afl->var_byte_count, afl->use_banner, - afl->unicorn_mode ? "unicorn" : "", afl->fsrv.qemu_mode ? "qemu " : "", - afl->non_instrumented_mode ? " non_instrumented " : "", - afl->no_forkserver ? "no_fsrv " : "", afl->crash_mode ? "crash " : "", - afl->persistent_mode ? "persistent " : "", - afl->shmem_testcase_mode ? "shmem_testcase " : "", - afl->deferred_mode ? "deferred " : "", - (afl->unicorn_mode || afl->fsrv.qemu_mode || afl->non_instrumented_mode || - afl->no_forkserver || afl->crash_mode || afl->persistent_mode || - afl->deferred_mode) - ? "" - : "default", - afl->orig_cmdline); + t_bytes, afl->var_byte_count, afl->use_banner, + afl->unicorn_mode ? "unicorn" : "", + afl->fsrv.qemu_mode ? "qemu " : "", + afl->non_instrumented_mode ? " non_instrumented " : "", + afl->no_forkserver ? "no_fsrv " : "", afl->crash_mode ? "crash " : "", + afl->persistent_mode ? "persistent " : "", + afl->shmem_testcase_mode ? "shmem_testcase " : "", + afl->deferred_mode ? "deferred " : "", + (afl->unicorn_mode || afl->fsrv.qemu_mode || + afl->non_instrumented_mode || afl->no_forkserver || + afl->crash_mode || afl->persistent_mode || afl->deferred_mode) + ? "" + : "default", + afl->orig_cmdline); /* ignore errors */ if (afl->debug) { diff --git a/test/test-floatingpoint.c b/test/test-floatingpoint.c index 083f0df5..acecd55a 100644 --- a/test/test-floatingpoint.c +++ b/test/test-floatingpoint.c @@ -8,16 +8,16 @@ __AFL_FUZZ_INIT(); int main(void) { ssize_t bytes_read; - + __AFL_INIT(); - float *magic = (float*)__AFL_FUZZ_TESTCASE_BUF; - + float *magic = (float *)__AFL_FUZZ_TESTCASE_BUF; + while (__AFL_LOOP(INT_MAX)) { if (__AFL_FUZZ_TESTCASE_LEN != sizeof(float)) return 1; /* 15 + 1/2 + 1/8 + 1/32 + 1/128 */ if ((-*magic == 15.0 + 0.5 + 0.125 + 0.03125 + 0.0078125)) abort(); - + } return 0; -- cgit 1.4.1 From 320f26d26f7e0cbe093e6f5af5f27f180bc31a1b Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 30 Jul 2020 19:00:41 +0200 Subject: add -b option to afl-fuzz --- docs/Changelog.md | 1 + include/afl-fuzz.h | 3 ++- src/afl-fuzz-init.c | 22 +++++++++++++++++----- src/afl-fuzz-state.c | 1 + src/afl-fuzz.c | 19 ++++++++++++++++--- 5 files changed, 37 insertions(+), 9 deletions(-) (limited to 'include/afl-fuzz.h') diff --git a/docs/Changelog.md b/docs/Changelog.md index 1e7a1c1d..dcaf64a7 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -15,6 +15,7 @@ sending a mail to . - afl-fuzz: - added -F option to allow -M main fuzzers to sync to foreign fuzzers, e.g. honggfuzz or libfuzzer + - added -b option to bind to a specific CPU - eliminated CPU affinity race condition for -S/-M runs - expanded havoc mode added, on no cycle finds add extra splicing and MOpt into the mix diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 1c1be711..bc3f65b6 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -545,7 +545,8 @@ typedef struct afl_state { u64 total_bitmap_size, /* Total bit count for all bitmaps */ total_bitmap_entries; /* Number of bitmaps counted */ - s32 cpu_core_count; /* CPU core count */ + s32 cpu_core_count, /* CPU core count */ + cpu_to_bind; /* bind to specific CPU */ #ifdef HAVE_AFFINITY s32 cpu_aff; /* Selected CPU core */ diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 65ad0c9f..ad92dff6 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -53,6 +53,13 @@ void bind_to_free_cpu(afl_state_t *afl) { u8 cpu_used[4096] = {0}, lockfile[PATH_MAX] = ""; u32 i; + if (afl->cpu_to_bind != -1) { + + i = afl->cpu_to_bind; + goto set_cpu; + + } + if (afl->sync_id) { s32 lockfd, first = 1; @@ -295,20 +302,23 @@ void bind_to_free_cpu(afl_state_t *afl) { try: + if (afl->cpu_to_bind != -1) + FATAL("bind to CPU #%d failed!", afl->cpu_to_bind); + #if !defined(__ANDROID__) - for (i = cpu_start; i < afl->cpu_core_count; i++) { + for (i = cpu_start; i < afl->cpu_core_count; i++) { - if (!cpu_used[i]) { break; } + if (!cpu_used[i]) { break; } - } + } if (i == afl->cpu_core_count) { #else - for (i = afl->cpu_core_count - cpu_start - 1; i > -1; i--) - if (!cpu_used[i]) break; + for (i = afl->cpu_core_count - cpu_start - 1; i > -1; i--) + if (!cpu_used[i]) break; if (i == -1) { #endif @@ -327,6 +337,8 @@ void bind_to_free_cpu(afl_state_t *afl) { OKF("Found a free CPU core, try binding to #%u.", i); +set_cpu: + afl->cpu_aff = i; #if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__) diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 66280ed1..e2d62bc6 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -94,6 +94,7 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->havoc_div = 1; /* Cycle count divisor for havoc */ afl->stage_name = "init"; /* Name of the current fuzz stage */ afl->splicing_with = -1; /* Splicing with which test case? */ + afl->cpu_to_bind = -1; #ifdef HAVE_AFFINITY afl->cpu_aff = -1; /* Selected CPU core */ diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 5bedf6e1..e33a4bbd 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -143,6 +143,8 @@ static void usage(afl_state_t *afl, u8 *argv0, int more_help) { //" -B bitmap.txt - mutate a specific test case, use the out/fuzz_bitmap //" "file\n" " -C - crash exploration mode (the peruvian rabbit thing)\n" + " -b cpu_id - bind the fuzzing process to the specified CPU core " + "(0-...)\n" " -e ext - file extension for the fuzz test input file (if " "needed)\n\n", argv0, EXEC_TIMEOUT, MEM_LIMIT, FOREIGN_SYNCS_MAX); @@ -271,9 +273,9 @@ int main(int argc, char **argv_orig, char **envp) { afl->shmem_testcase_mode = 1; // we always try to perform shmem fuzzing - while ((opt = getopt(argc, argv, - "+c:i:I:o:f:F:m:t:T:dDnCB:S:M:x:QNUWe:p:s:V:E:L:hRP:")) > - 0) { + while ((opt = getopt( + argc, argv, + "+b:c:i:I:o:f:F:m:t:T:dDnCB:S:M:x:QNUWe:p:s:V:E:L:hRP:")) > 0) { switch (opt) { @@ -281,6 +283,17 @@ int main(int argc, char **argv_orig, char **envp) { afl->infoexec = optarg; break; + case 'b': { /* bind CPU core */ + + if (afl->cpu_to_bind != -1) FATAL("Multiple -b options not supported"); + + if (sscanf(optarg, "%u", &afl->cpu_to_bind) < 0 || optarg[0] == '-') + FATAL("Bad syntax used for -b"); + + break; + + } + case 'c': { afl->shm.cmplog_mode = 1; -- cgit 1.4.1 From 54d96685808bd7b47de30f5028727d94b5369962 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 31 Jul 2020 17:21:17 +0000 Subject: Haiku set explicitly to performance. No command line to set through afl-system-config (the only one is a GUI). --- include/afl-fuzz.h | 5 +++++ src/afl-fuzz.c | 5 +++++ 2 files changed, 10 insertions(+) (limited to 'include/afl-fuzz.h') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index bc3f65b6..b82ddb4a 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -82,6 +82,11 @@ #include #endif /* __APPLE__ || __FreeBSD__ || __OpenBSD__ */ +#if defined(__HAIKU__) + #include + #include +#endif + /* For systems that have sched_setaffinity; right now just Linux, but one can hope... */ diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 54db1efb..326ccc1c 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1057,6 +1057,11 @@ int main(int argc, char **argv_orig, char **envp) { bind_to_free_cpu(afl); #endif /* HAVE_AFFINITY */ + #ifdef __HAIKU__ + /* Prioritizes performance over power saving */ + set_scheduler_mode(SCHEDULER_MODE_LOW_LATENCY); + #endif + afl->fsrv.trace_bits = afl_shm_init(&afl->shm, afl->fsrv.map_size, afl->non_instrumented_mode); -- cgit 1.4.1