From e1d5009229fb5cea5845cd08e0abdc8fe440ee86 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 9 Jul 2021 10:32:14 +0200 Subject: fixes --- custom_mutators/gramatron/gramfuzz.h | 253 +++++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 custom_mutators/gramatron/gramfuzz.h (limited to 'custom_mutators/gramatron/gramfuzz.h') diff --git a/custom_mutators/gramatron/gramfuzz.h b/custom_mutators/gramatron/gramfuzz.h new file mode 100644 index 00000000..811e0af7 --- /dev/null +++ b/custom_mutators/gramatron/gramfuzz.h @@ -0,0 +1,253 @@ +#ifndef _GRAMFUZZ_H + + #define _GRAMFUZZ_H + + #include + #include + #include "hashmap.h" + #include "uthash.h" + #include "utarray.h" + + #define INIT_INPUTS 100 // No. of initial inputs to be generated + +// Set this as `numstates` + 1 where `numstates` is retrieved from gen automata +// json #define STATES 63 + + #define INIT_SIZE 100 // Initial size of the dynamic array holding the input + + #define SPLICE_CORPUS 10000 + #define RECUR_THRESHOLD 6 + #define SIZE_THRESHOLD 2048 + + #define FLUSH_INTERVAL \ + 3600 // Inputs that gave new coverage will be dumped every FLUSH_INTERVAL + // seconds + +typedef struct trigger { + + char * id; + int dest; + char * term; + size_t term_len; + +} trigger; + +typedef struct state { + + int state_name; // Integer State name + int trigger_len; // Number of triggers associated with this state + trigger *ptr; // Pointer to beginning of the list of triggers + +} state; + +typedef struct terminal { + + int state; + int trigger_idx; + size_t symbol_len; + char * symbol; + +} terminal; + +typedef struct buckethash { + + int freq; + +} buckethash; + +int init_state; +int curr_state; +int final_state; +int numstates; + +/***************** +/ DYNAMIC ARRAY FOR WALKS +*****************/ + +typedef struct { + + size_t used; + size_t size; + size_t inputlen; + terminal *start; + +} Array; + +/***************** +/ DYNAMIC ARRAY FOR STATEMAPS/RECURSION MAPS +*****************/ + +typedef struct { + + int * array; + size_t used; + size_t size; + +} IdxMap; + +typedef struct { + + UT_array *nums; + +} IdxMap_new; + +typedef struct { + + IdxMap_new *idxmap; + UT_array ** recurIdx; + +} Get_Dupes_Ret; + +/* Candidate Struct */ +typedef struct { + + Array * walk; + IdxMap_new *statemap; + +} Candidate; + +/* Splice Mutation helpers*/ +typedef struct { + + Candidate *splice_cand; + int idx; + +} SpliceCand; + +typedef struct { + + SpliceCand *start; + size_t used; + size_t size; + +} SpliceCandArray; + +// Initialize dynamic array for potential splice points +SpliceCand potential[SPLICE_CORPUS]; + +typedef struct { + + int orig_idx; + int splice_idx; + +} intpair_t; + +// Initialize dynamic array for potential splice points +// SpliceCand potential[SPLICE_CORPUS]; +// IdxMap_new* rcuridx[STATES]; + +/* Prototypes*/ +Array * slice(Array *, int); +state * create_pda(u8 *); +Array * gen_input(state *, Array *); +Array * gen_input_count(state *, Array *, int *); +int updatebucket(map_t, int); +void itoa(int, char *, int); +void strrreverse(char *, char *); +void dbg_hashmap(map_t); +void print_repr(Array *, char *); +int isSatisfied(map_t); +char * get_state(char *); +Candidate *gen_candidate(Array *); + +Array *spliceGF(Array *, Array *, int); +Array *performSpliceOne(Array *, IdxMap_new *, Array *); +/* Mutation Methods*/ +Array * performRandomMutation(state *, Array *); +Array * performRandomMutationCount(state *, Array *, int *); +Array * performSpliceMutationBench(state *, Array *, Candidate **); +UT_array **get_dupes(Array *, int *); +Array * doMult(Array *, UT_array **, int); +Array * doMultBench(Array *, UT_array **, int); + +/* Benchmarks*/ +void SpaceBenchmark(char *); +void GenInputBenchmark(char *, char *); +void RandomMutationBenchmark(char *, char *); +void MutationAggrBenchmark(char *, char *); +void SpliceMutationBenchmark(char *, char *); +void SpliceMutationBenchmarkOne(char *, char *); +void RandomRecursiveBenchmark(char *, char *); + +/* Testers */ +void SanityCheck(char *); + +/*Helpers*/ +void initArray(Array *, size_t); +void insertArray(Array *, int, char *, size_t, int); +void freeArray(Array *); +void initArrayIdx(IdxMap *, size_t); +void insertArrayIdx(IdxMap *, int); +void freeArrayIdx(IdxMap *); +void initArraySplice(SpliceCandArray *, size_t); +void insertArraySplice(SpliceCandArray *, Candidate *, int); +void freeArraySplice(IdxMap *); +void getTwoIndices(UT_array *, int, int *, int *); +void swap(int *, int *); +Array *slice_inverse(Array *, int); +void concatPrefixFeature(Array *, Array *); +void concatPrefixFeatureBench(Array *, Array *); +Array *carve(Array *, int, int); +int fact(int); + +void add_to_corpus(struct json_object *, Array *); +struct json_object *term_to_json(terminal *); + +/* Gramatron specific prototypes */ +u8 * unparse_walk(Array *); +Array *performSpliceGF(state *, Array *, afl_state_t *); +void dump_input(u8 *, char *, int *); +void write_input(Array *, u8 *); +Array *read_input(state *, u8 *); +state *pda; + +// // AFL-specific struct +// typedef uint8_t u8; +// typedef uint16_t u16; +// typedef uint32_t u32; +// #ifdef __x86_64__ +// typedef unsigned long long u64; +// #else +// typedef uint64_t u64; +// #endif /* ^__x86_64__ */ +// +// struct queue_entry { + +// Array* walk; /* Pointer to the automaton walk*/ +// u32 walk_len; /* Number of tokens in the input*/ +// Candidate* cand; /* Preprocessed info about the +// candidate to allow for faster mutations*/ +// +// u8* fname; /* File name for the test case */ +// u32 len; /* Input length */ +// UT_array** recur_idx; /* Keeps track of recursive feature +// indices*/ +// +// u32 recur_len; /* The number of recursive features*/ +// +// u8 cal_failed, /* Calibration failed? */ +// trim_done, /* Trimmed? */ +// was_fuzzed, /* Had any fuzzing done yet? */ +// passed_det, /* Deterministic stages passed? */ +// has_new_cov, /* Triggers new coverage? */ +// var_behavior, /* Variable behavior? */ +// favored, /* Currently favored? */ +// fs_redundant; /* Marked as redundant in the fs? */ +// +// u32 bitmap_size, /* Number of bits set in bitmap */ +// exec_cksum; /* Checksum of the execution trace */ +// +// u64 exec_us, /* Execution time (us) */ +// handicap, /* Number of queue cycles behind */ +// depth; /* Path depth */ +// +// u8* trace_mini; /* Trace bytes, if kept */ +// u32 tc_ref; /* Trace bytes ref count */ +// +// struct queue_entry *next, /* Next element, if any */ +// *next_100; /* 100 elements ahead */ +// +// }; + +#endif + -- cgit 1.4.1 From d354ec2586a3a31c87a8b95433c2886f04c44a03 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 9 Jul 2021 11:39:25 +0200 Subject: more fixes --- custom_mutators/gramatron/README.md | 4 +- custom_mutators/gramatron/gramfuzz-util.c | 70 +++++++++++++++++++++++++------ custom_mutators/gramatron/gramfuzz.c | 28 +++++++------ custom_mutators/gramatron/gramfuzz.h | 29 ++++++------- custom_mutators/gramatron/uthash.h | 25 +++++++++-- include/afl-fuzz.h | 2 + src/afl-fuzz-init.c | 12 +++--- src/afl-fuzz-mutators.c | 39 +++++++++++++++++ src/afl-fuzz-queue.c | 38 ++--------------- 9 files changed, 162 insertions(+), 85 deletions(-) (limited to 'custom_mutators/gramatron/gramfuzz.h') diff --git a/custom_mutators/gramatron/README.md b/custom_mutators/gramatron/README.md index 7f73cf2c..6659cb95 100644 --- a/custom_mutators/gramatron/README.md +++ b/custom_mutators/gramatron/README.md @@ -23,8 +23,8 @@ You have to set the grammar file to use with `GRAMMATRON_AUTOMATION`: ``` export AFL_DISABLE_TRIM=1 export AFL_CUSTOM_MUTATOR_ONLY=1 -export AFL_CUSTOM_MUTATOR_LIBRARY=./grammatron.so -export GRAMMATRON_AUTOMATION=grammars/ruby/source_automata.json +export AFL_CUSTOM_MUTATOR_LIBRARY=./gramatron.so +export GRAMATRON_AUTOMATION=grammars/ruby/source_automata.json afl-fuzz -i in -o out -- ./target ``` diff --git a/custom_mutators/gramatron/gramfuzz-util.c b/custom_mutators/gramatron/gramfuzz-util.c index cb2e1b59..41ffd86d 100644 --- a/custom_mutators/gramatron/gramfuzz-util.c +++ b/custom_mutators/gramatron/gramfuzz-util.c @@ -4,6 +4,11 @@ #include #include "afl-fuzz.h" #include "gramfuzz.h" +#ifdef _GNU_SOURCE + #undef _GNU_SOURCE +#endif +#define _GNU_SOURCE +#include /* Dynamic Array for adding to the input repr * */ @@ -178,7 +183,7 @@ void write_input(Array *input, u8 *fn) { // If the input has already been flushed, then skip silently if (fp == NULL) { - printf("\n File could not be open, exiting"); + fprintf(stderr, "\n File '%s' could not be open, exiting\n", fn); exit(1); } @@ -196,22 +201,13 @@ void write_input(Array *input, u8 *fn) { } -// Read the input representation into memory -Array *read_input(state *pda, u8 *fn) { +Array *parse_input(state *pda, FILE *fp) { - FILE * fp; terminal *term; state * state_ptr; trigger * trigger; int trigger_idx; Array * input = (Array *)calloc(1, sizeof(Array)); - fp = fopen(fn, "rb"); - if (fp == NULL) { - - printf("\nFile:%s does not exist..exiting", fn); - exit(1); - - } // Read the length parameters fread(&input->used, sizeof(size_t), 1, fp); @@ -219,6 +215,12 @@ Array *read_input(state *pda, u8 *fn) { fread(&input->inputlen, sizeof(size_t), 1, fp); terminal *start_ptr = (terminal *)calloc(input->size, sizeof(terminal)); + if (!start_ptr) { + + fprintf(stderr, "alloc failed!\n"); + return NULL; + + } // Read the dynamic array to memory fread(start_ptr, input->size * sizeof(terminal), 1, fp); @@ -242,9 +244,51 @@ Array *read_input(state *pda, u8 *fn) { // printf("\nUsed:%zu Size:%zu Inputlen:%zu", input->used, input->size, // input->inputlen); - fclose(fp); - return input; } +Array *open_input(state *pda, u8 *data, size_t len) { + + int fd = memfd_create("foo", O_RDWR); + if (fd < 0) { + + fprintf(stderr, "Error: memfd_create failed\n"); + return NULL; + + } + + ck_write(fd, data, len, "memfd_create"); + lseek(fd, 0, SEEK_SET); + FILE *f = fdopen(fd, "rb"); + if (!f) { + + fprintf(stderr, "Error: fdopen failed\n"); + return NULL; + + } + + Array *res = parse_input(pda, f); + fclose(f); + return res; + +} + +// Read the input representation into memory +Array *read_input(state *pda, u8 *fn) { + + FILE *fp; + fp = fopen(fn, "rb"); + if (fp == NULL) { + + fprintf(stderr, "\n File '%s' does not exist, exiting\n", fn); + exit(1); + + } + + Array *res = parse_input(pda, fp); + fclose(fp); + return res; + +} + diff --git a/custom_mutators/gramatron/gramfuzz.c b/custom_mutators/gramatron/gramfuzz.c index 5c96ddce..55b631e6 100644 --- a/custom_mutators/gramatron/gramfuzz.c +++ b/custom_mutators/gramatron/gramfuzz.c @@ -159,7 +159,7 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { // u32 recur_len = 0; // The number of recursive features // data->mutator_buf = NULL; - char *automaton_file = getenv("GRAMMATRON_AUTOMATION"); + char *automaton_file = getenv("GRAMATRON_AUTOMATION"); if (automaton_file) { pda = create_pda(automaton_file); @@ -168,7 +168,7 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { fprintf(stderr, "\nError: GrammaTron needs an automation json file set in " - "AFL_GRAMMATRON_AUTOMATON\n"); + "AFL_GRAMATRON_AUTOMATON\n"); exit(-1); } @@ -208,18 +208,18 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, doMult(data->orig_walk, data->recurIdx, data->recurlen); data->mut_alloced = 1; - } else if (data->mut_idx == 2) { // Perform splice mutation + /*} else if (data->mut_idx == 2) { // Perform splice mutation - // Read the input representation for the splice candidate - u8 * automaton_fn = alloc_printf("%s.aut", add_buf); - Array *spliceCandidate = read_input(pda, automaton_fn); + // Read the input representation for the splice candidate + //u8 * automaton_fn = alloc_printf("%s.aut", add_buf); + Array *spliceCandidate = open_input(pda, add_buf, add_buf_size); - data->mutated_walk = - performSpliceOne(data->orig_walk, data->statemap, spliceCandidate); - data->mut_alloced = 1; - free(spliceCandidate->start); - free(spliceCandidate); - ck_free(automaton_fn); + data->mutated_walk = + performSpliceOne(data->orig_walk, data->statemap, spliceCandidate); + data->mut_alloced = 1; + free(spliceCandidate->start); + free(spliceCandidate); + //ck_free(automaton_fn);*/ } else { // Generate an input from scratch @@ -262,6 +262,10 @@ u8 afl_custom_queue_new_entry(my_mutator_t * data, automaton_fn = alloc_printf("%s.aut", filename_new_queue); // Check if this method is being called during initialization + + // fprintf(stderr, "new: %s, old: %s, auto: %s\n", + // filename_new_queue,filename_orig_queue,automaton_fn); + if (filename_orig_queue) { write_input(data->mutated_walk, automaton_fn); diff --git a/custom_mutators/gramatron/gramfuzz.h b/custom_mutators/gramatron/gramfuzz.h index 811e0af7..46cde8ec 100644 --- a/custom_mutators/gramatron/gramfuzz.h +++ b/custom_mutators/gramatron/gramfuzz.h @@ -1,27 +1,27 @@ #ifndef _GRAMFUZZ_H - #define _GRAMFUZZ_H +#define _GRAMFUZZ_H - #include - #include - #include "hashmap.h" - #include "uthash.h" - #include "utarray.h" +#include +#include +#include "hashmap.h" +#include "uthash.h" +#include "utarray.h" - #define INIT_INPUTS 100 // No. of initial inputs to be generated +#define INIT_INPUTS 100 // No. of initial inputs to be generated // Set this as `numstates` + 1 where `numstates` is retrieved from gen automata // json #define STATES 63 - #define INIT_SIZE 100 // Initial size of the dynamic array holding the input +#define INIT_SIZE 100 // Initial size of the dynamic array holding the input - #define SPLICE_CORPUS 10000 - #define RECUR_THRESHOLD 6 - #define SIZE_THRESHOLD 2048 +#define SPLICE_CORPUS 10000 +#define RECUR_THRESHOLD 6 +#define SIZE_THRESHOLD 2048 - #define FLUSH_INTERVAL \ - 3600 // Inputs that gave new coverage will be dumped every FLUSH_INTERVAL - // seconds +#define FLUSH_INTERVAL \ + 3600 // Inputs that gave new coverage will be dumped every FLUSH_INTERVAL + // seconds typedef struct trigger { @@ -199,6 +199,7 @@ Array *performSpliceGF(state *, Array *, afl_state_t *); void dump_input(u8 *, char *, int *); void write_input(Array *, u8 *); Array *read_input(state *, u8 *); +Array *open_input(state *, u8 *, size_t); state *pda; // // AFL-specific struct diff --git a/custom_mutators/gramatron/uthash.h b/custom_mutators/gramatron/uthash.h index 5957899a..05c8abe6 100644 --- a/custom_mutators/gramatron/uthash.h +++ b/custom_mutators/gramatron/uthash.h @@ -59,6 +59,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *_da_dst = (char *)(src); \ \ } while (0) + #else #define DECLTYPE_ASSIGN(dst, src) \ do { \ @@ -66,6 +67,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. (dst) = DECLTYPE(dst)(src); \ \ } while (0) + #endif /* a number of the hash function use uint32_t which isn't defined on Pre VS2010 @@ -138,6 +140,7 @@ typedef unsigned char uint8_t; (oomed) = 1; \ \ } while (0) +\ #define IF_HASH_NONFATAL_OOM(x) x #else @@ -153,10 +156,11 @@ typedef unsigned char uint8_t; #endif /* initial number of buckets */ -#define HASH_INITIAL_NUM_BUCKETS 32U /* initial number of buckets */ -#define HASH_INITIAL_NUM_BUCKETS_LOG2 5U /* lg2 of initial number of buckets \ - */ -#define HASH_BKT_CAPACITY_THRESH 10U /* expand when bucket count reaches */ +#define HASH_INITIAL_NUM_BUCKETS 32U /* initial number of buckets */ +#define HASH_INITIAL_NUM_BUCKETS_LOG2 \ + 5U /* lg2 of initial number of buckets \ + */ +#define HASH_BKT_CAPACITY_THRESH 10U /* expand when bucket count reaches */ /* calculate the element whose hash handle address is hhp */ #define ELMT_FROM_HH(tbl, hhp) ((void *)(((char *)(hhp)) - ((tbl)->hho))) @@ -376,6 +380,8 @@ typedef unsigned char uint8_t; \ } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \ \ + \ + \ } while (0) #ifdef NO_DECLTYPE @@ -397,6 +403,8 @@ typedef unsigned char uint8_t; \ } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \ \ + \ + \ } while (0) #endif @@ -639,6 +647,7 @@ typedef unsigned char uint8_t; HASH_FIND(hh, head, findstr, _uthash_hfstr_keylen, out); \ \ } while (0) +\ #define HASH_ADD_STR(head, strfield, add) \ do { \ \ @@ -646,6 +655,7 @@ typedef unsigned char uint8_t; HASH_ADD(hh, head, strfield[0], _uthash_hastr_keylen, add); \ \ } while (0) +\ #define HASH_REPLACE_STR(head, strfield, add, replaced) \ do { \ \ @@ -653,6 +663,7 @@ typedef unsigned char uint8_t; HASH_REPLACE(hh, head, strfield[0], _uthash_hrstr_keylen, add, replaced); \ \ } while (0) +\ #define HASH_FIND_INT(head, findint, out) \ HASH_FIND(hh, head, findint, sizeof(int), out) #define HASH_ADD_INT(head, intfield, add) \ @@ -679,6 +690,7 @@ typedef unsigned char uint8_t; exit(-1); \ \ } while (0) +\ #define HASH_FSCK(hh, head, where) \ do { \ \ @@ -748,6 +760,7 @@ typedef unsigned char uint8_t; } \ \ } while (0) + #else #define HASH_FSCK(hh, head, where) #endif @@ -764,6 +777,7 @@ typedef unsigned char uint8_t; write(HASH_EMIT_KEYS, keyptr, (unsigned long)fieldlen); \ \ } while (0) + #else #define HASH_EMIT_KEY(hh, head, keyptr, fieldlen) #endif @@ -806,6 +820,7 @@ typedef unsigned char uint8_t; } \ \ } while (0) + /* FNV-1a variation */ #define HASH_FNV(key, keylen, hashv) \ do { \ @@ -1098,6 +1113,7 @@ typedef unsigned char uint8_t; hashv = _mur_h1; \ \ } while (0) + #endif /* HASH_USING_NO_STRICT_ALIASING */ /* iterate over items in a known bucket to find desired item */ @@ -1335,6 +1351,7 @@ typedef unsigned char uint8_t; _hs_psize--; \ \ } else if ((cmpfcn(DECLTYPE(head)( \ + \ ELMT_FROM_HH((head)->hh.tbl, _hs_p)), \ DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl, \ _hs_q)))) <= 0) { \ diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 70d9473e..16409892 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1005,6 +1005,8 @@ void setup_custom_mutators(afl_state_t *); void destroy_custom_mutators(afl_state_t *); u8 trim_case_custom(afl_state_t *, struct queue_entry *q, u8 *in_buf, struct custom_mutator *mutator); +void run_afl_custom_queue_new_entry(afl_state_t *, struct queue_entry *, u8 *, + u8 *); /* Python */ #ifdef USE_PYTHON diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 5e4f1585..faa45a4e 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -881,11 +881,7 @@ void perform_dry_run(afl_state_t *afl) { u32 read_len = MIN(q->len, (u32)MAX_FILE); use_mem = afl_realloc(AFL_BUF_PARAM(in), read_len); - if (read(fd, use_mem, read_len) != (ssize_t)read_len) { - - FATAL("Short read from '%s'", q->fname); - - } + ck_read(fd, use_mem, read_len, q->fname); close(fd); @@ -1350,6 +1346,12 @@ void pivot_inputs(afl_state_t *afl) { if (q->passed_det) { mark_as_det_done(afl, q); } + if (afl->custom_mutators_count) { + + run_afl_custom_queue_new_entry(afl, q, q->fname, NULL); + + } + ++id; } diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index e27d6fae..91bae48e 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -31,6 +31,45 @@ struct custom_mutator *load_custom_mutator(afl_state_t *, const char *); struct custom_mutator *load_custom_mutator_py(afl_state_t *, char *); #endif +void run_afl_custom_queue_new_entry(afl_state_t *afl, struct queue_entry *q, + u8 *fname, u8 *mother_fname) { + + if (afl->custom_mutators_count) { + + u8 updated = 0; + + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_queue_new_entry) { + + if (el->afl_custom_queue_new_entry(el->data, fname, mother_fname)) { + + updated = 1; + + } + + } + + }); + + if (updated) { + + struct stat st; + if (stat(fname, &st)) { PFATAL("File %s is gone!", fname); } + if (!st.st_size) { + + FATAL("File %s became empty in custom mutator!", fname); + + } + + q->len = st.st_size; + + } + + } + +} + void setup_custom_mutators(afl_state_t *afl) { /* Try mutator library first */ diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 48794e95..8080775f 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -474,42 +474,10 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { if (afl->custom_mutators_count) { - u8 updated = 0; + /* At the initialization stage, queue_cur is NULL */ + if (afl->queue_cur && !afl->syncing_party) { - LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { - - if (el->afl_custom_queue_new_entry) { - - u8 *fname_orig = NULL; - - /* At the initialization stage, queue_cur is NULL */ - if (afl->queue_cur && !afl->syncing_party) { - - fname_orig = afl->queue_cur->fname; - - } - - if (el->afl_custom_queue_new_entry(el->data, fname, fname_orig)) { - - updated = 1; - - } - - } - - }); - - if (updated) { - - struct stat st; - if (stat(fname, &st)) { PFATAL("File %s is gone!", fname); } - if (!st.st_size) { - - FATAL("File %s became empty in custom mutator!", fname); - - } - - q->len = st.st_size; + run_afl_custom_queue_new_entry(afl, q, fname, afl->queue_cur->fname); } -- cgit 1.4.1 From 1b2637545b6b9f171ea46ed6cf5164e8d05d1776 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 9 Jul 2021 12:31:29 +0200 Subject: reenable splicing --- custom_mutators/gramatron/gramfuzz-util.c | 26 -------------------------- custom_mutators/gramatron/gramfuzz.c | 24 +++++++++++++++++++----- custom_mutators/gramatron/gramfuzz.h | 1 - 3 files changed, 19 insertions(+), 32 deletions(-) (limited to 'custom_mutators/gramatron/gramfuzz.h') diff --git a/custom_mutators/gramatron/gramfuzz-util.c b/custom_mutators/gramatron/gramfuzz-util.c index 41ffd86d..2e0af936 100644 --- a/custom_mutators/gramatron/gramfuzz-util.c +++ b/custom_mutators/gramatron/gramfuzz-util.c @@ -248,32 +248,6 @@ Array *parse_input(state *pda, FILE *fp) { } -Array *open_input(state *pda, u8 *data, size_t len) { - - int fd = memfd_create("foo", O_RDWR); - if (fd < 0) { - - fprintf(stderr, "Error: memfd_create failed\n"); - return NULL; - - } - - ck_write(fd, data, len, "memfd_create"); - lseek(fd, 0, SEEK_SET); - FILE *f = fdopen(fd, "rb"); - if (!f) { - - fprintf(stderr, "Error: fdopen failed\n"); - return NULL; - - } - - Array *res = parse_input(pda, f); - fclose(f); - return res; - -} - // Read the input representation into memory Array *read_input(state *pda, u8 *fn) { diff --git a/custom_mutators/gramatron/gramfuzz.c b/custom_mutators/gramatron/gramfuzz.c index 55b631e6..fd126ec0 100644 --- a/custom_mutators/gramatron/gramfuzz.c +++ b/custom_mutators/gramatron/gramfuzz.c @@ -208,18 +208,32 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, doMult(data->orig_walk, data->recurIdx, data->recurlen); data->mut_alloced = 1; - /*} else if (data->mut_idx == 2) { // Perform splice mutation + } else if (data->mut_idx == 2) { // Perform splice mutation - // Read the input representation for the splice candidate - //u8 * automaton_fn = alloc_printf("%s.aut", add_buf); - Array *spliceCandidate = open_input(pda, add_buf, add_buf_size); + // we cannot use the supplied splice data so choose a new random file + u32 tid = rand() % data->afl->queued_paths; + struct queue_entry *q = data->afl->queue_buf[tid]; + + // Read the input representation for the splice candidate + u8 * automaton_fn = alloc_printf("%s.aut", q->fname); + Array *spliceCandidate = read_input(pda, automaton_fn); + + if (spliceCandidate) { data->mutated_walk = performSpliceOne(data->orig_walk, data->statemap, spliceCandidate); data->mut_alloced = 1; free(spliceCandidate->start); free(spliceCandidate); - //ck_free(automaton_fn);*/ + + } else { + + data->mutated_walk = gen_input(pda, NULL); + data->mut_alloced = 1; + + } + + ck_free(automaton_fn); } else { // Generate an input from scratch diff --git a/custom_mutators/gramatron/gramfuzz.h b/custom_mutators/gramatron/gramfuzz.h index 46cde8ec..e6912074 100644 --- a/custom_mutators/gramatron/gramfuzz.h +++ b/custom_mutators/gramatron/gramfuzz.h @@ -199,7 +199,6 @@ Array *performSpliceGF(state *, Array *, afl_state_t *); void dump_input(u8 *, char *, int *); void write_input(Array *, u8 *); Array *read_input(state *, u8 *); -Array *open_input(state *, u8 *, size_t); state *pda; // // AFL-specific struct -- cgit 1.4.1 From 3101e9c88df72755b461e82870879fbe9e7429fa Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 20 Jul 2021 11:10:58 +0200 Subject: add gramatron --- .gitmodules | 3 + custom_mutators/gramatron/JSONC_VERSION | 1 + custom_mutators/gramatron/Makefile | 7 -- custom_mutators/gramatron/README.md | 18 +-- .../gramatron/build_gramatron_mutator.sh | 140 +++++++++++++++++++++ custom_mutators/gramatron/gramfuzz-helpers.c | 6 +- custom_mutators/gramatron/gramfuzz-mutators.c | 11 +- custom_mutators/gramatron/gramfuzz.c | 4 +- custom_mutators/gramatron/gramfuzz.h | 2 + custom_mutators/gramatron/json-c | 1 + docs/Changelog.md | 2 +- 11 files changed, 164 insertions(+), 31 deletions(-) create mode 100644 custom_mutators/gramatron/JSONC_VERSION delete mode 100644 custom_mutators/gramatron/Makefile create mode 100755 custom_mutators/gramatron/build_gramatron_mutator.sh create mode 160000 custom_mutators/gramatron/json-c (limited to 'custom_mutators/gramatron/gramfuzz.h') diff --git a/.gitmodules b/.gitmodules index c787ec0e..6edefd72 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "qemu_mode/qemuafl"] path = qemu_mode/qemuafl url = https://github.com/AFLplusplus/qemuafl +[submodule "custom_mutators/gramatron/json-c"] + path = custom_mutators/gramatron/json-c + url = https://github.com/json-c/json-c diff --git a/custom_mutators/gramatron/JSONC_VERSION b/custom_mutators/gramatron/JSONC_VERSION new file mode 100644 index 00000000..7663833a --- /dev/null +++ b/custom_mutators/gramatron/JSONC_VERSION @@ -0,0 +1 @@ +af8dd4a307e7b837f9fa2959549548ace4afe08b diff --git a/custom_mutators/gramatron/Makefile b/custom_mutators/gramatron/Makefile deleted file mode 100644 index d24f3dd4..00000000 --- a/custom_mutators/gramatron/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -all: gramatron.so - -gramatron.so: gramfuzz.c gramfuzz.h gramfuzz-helpers.c gramfuzz-mutators.c gramfuzz-util.c hashmap.c hashmap.h test.c test.h utarray.h uthash.h - $(CC) -O3 -g -fPIC -Wno-unused-result -Wl,--allow-multiple-definition -I../../include -o gramatron.so -shared -I. -I/prg/dev/include gramfuzz.c gramfuzz-helpers.c gramfuzz-mutators.c gramfuzz-util.c hashmap.c test.c -ljson-c - -clean: - rm -f gramatron.so diff --git a/custom_mutators/gramatron/README.md b/custom_mutators/gramatron/README.md index 6659cb95..91f93355 100644 --- a/custom_mutators/gramatron/README.md +++ b/custom_mutators/gramatron/README.md @@ -1,20 +1,15 @@ # GramaTron Gramatron is a coverage-guided fuzzer that uses grammar automatons to perform -grammar-aware fuzzing. Technical details about our framework are available in our -[ISSTA'21 paper](https://nebelwelt.net/files/21ISSTA.pdf). The artifact to reproduce the -experiments presented in our paper are present in `artifact/`. Instructions to run -a sample campaign and incorporate new grammars is presented below: +grammar-aware fuzzing. Technical details about our framework are available +in the [ISSTA'21 paper](https://nebelwelt.net/files/21ISSTA.pdf). +The artifact to reproduce the experiments presented in the paper are present +in `artifact/`. Instructions to run a sample campaign and incorporate new +grammars is presented below: # Compiling -- Install `json-c` -``` -git clone https://github.com/json-c/json-c.git -cd json-c && git reset --hard af8dd4a307e7b837f9fa2959549548ace4afe08b && sh autogen.sh && ./configure && make && make install -``` - -afterwards you can just `make` GrammaTron +Simply execute `./build_gramatron_mutator.sh` # Running @@ -48,4 +43,3 @@ Eg. ./preprocess/prep_automaton.sh ~/grammars/ruby/source.json PROGRAM Eg. ./test SanityCheck ~/grammars/ruby/source_automata.json ``` - diff --git a/custom_mutators/gramatron/build_gramatron_mutator.sh b/custom_mutators/gramatron/build_gramatron_mutator.sh new file mode 100755 index 00000000..c1cdf0e7 --- /dev/null +++ b/custom_mutators/gramatron/build_gramatron_mutator.sh @@ -0,0 +1,140 @@ +#!/bin/sh +# +# american fuzzy lop++ - gramatron build script +# ------------------------------------------------ +# +# Originally written by Nathan Voss +# +# Adapted from code by Andrew Griffiths and +# Michal Zalewski +# +# Adapted for AFLplusplus by Dominik Maier +# +# Copyright 2017 Battelle Memorial Institute. All rights reserved. +# Copyright 2019-2020 AFLplusplus Project. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# This script downloads, patches, and builds a version of Unicorn with +# minor tweaks to allow Unicorn-emulated binaries to be run under +# afl-fuzz. +# +# The modifications reside in patches/*. The standalone Unicorn library +# will be written to /usr/lib/libunicornafl.so, and the Python bindings +# will be installed system-wide. +# +# You must make sure that Unicorn Engine is not already installed before +# running this script. If it is, please uninstall it first. + +JSONC_VERSION="$(cat ./JSONC_VERSION)" +JSONC_REPO="https://github.com/json-c/json-c" + +echo "=================================================" +echo "Gramatron Mutator build script" +echo "=================================================" +echo + +echo "[*] Performing basic sanity checks..." + +PLT=`uname -s` + +if [ ! -f "../../config.h" ]; then + + echo "[-] Error: key files not found - wrong working directory?" + exit 1 + +fi + +PYTHONBIN=`command -v python3 || command -v python || command -v python2 || echo python3` +MAKECMD=make +TARCMD=tar + +if [ "$PLT" = "Darwin" ]; then + CORES=`sysctl -n hw.ncpu` + TARCMD=tar +fi + +if [ "$PLT" = "FreeBSD" ]; then + MAKECMD=gmake + CORES=`sysctl -n hw.ncpu` + TARCMD=gtar +fi + +if [ "$PLT" = "NetBSD" ] || [ "$PLT" = "OpenBSD" ]; then + MAKECMD=gmake + CORES=`sysctl -n hw.ncpu` + TARCMD=gtar +fi + +PREREQ_NOTFOUND= +for i in git $MAKECMD $TARCMD; do + + T=`command -v "$i" 2>/dev/null` + + if [ "$T" = "" ]; then + + echo "[-] Error: '$i' not found. Run 'sudo apt-get install $i' or similar." + PREREQ_NOTFOUND=1 + + fi + +done + +test -z "$CC" && export CC=cc + +if echo "$CC" | grep -qF /afl-; then + + echo "[-] Error: do not use afl-gcc or afl-clang to compile this tool." + PREREQ_NOTFOUND=1 + +fi + +if [ "$PREREQ_NOTFOUND" = "1" ]; then + exit 1 +fi + +echo "[+] All checks passed!" + +echo "[*] Making sure json-c is checked out" + +git status 1>/dev/null 2>/dev/null +if [ $? -eq 0 ]; then + echo "[*] initializing json-c submodule" + git submodule init || exit 1 + git submodule update ./json-c 2>/dev/null # ignore errors +else + echo "[*] cloning json-c" + test -d json-c || { + CNT=1 + while [ '!' -d json-c -a "$CNT" -lt 4 ]; do + echo "Trying to clone json-c (attempt $CNT/3)" + git clone "$JSONC_REPO" + CNT=`expr "$CNT" + 1` + done + } +fi + +test -d json-c || { echo "[-] not checked out, please install git or check your internet connection." ; exit 1 ; } +echo "[+] Got json-c." + +cd "json-c" || exit 1 +echo "[*] Checking out $JSONC_VERSION" +sh -c 'git stash && git stash drop' 1>/dev/null 2>/dev/null +git checkout "$JSONC_VERSION" || exit 1 +sh autogen.sh || exit 1 +export CFLAGS=-fPIC +./configure --disable-shared || exit 1 +make || exit 1 +cd .. + +echo +echo +echo "[+] Json-c successfully prepared!" +echo "[+] Builing gramatron now." +$CC -O3 -g -fPIC -Wno-unused-result -Wl,--allow-multiple-definition -I../../include -o gramatron.so -shared -I. -I/prg/dev/include gramfuzz.c gramfuzz-helpers.c gramfuzz-mutators.c gramfuzz-util.c hashmap.c json-c/.libs/libjson-c.a || exit 1 +echo +echo "[+] gramatron successfully built!" diff --git a/custom_mutators/gramatron/gramfuzz-helpers.c b/custom_mutators/gramatron/gramfuzz-helpers.c index f894c850..378a3d90 100644 --- a/custom_mutators/gramatron/gramfuzz-helpers.c +++ b/custom_mutators/gramatron/gramfuzz-helpers.c @@ -73,7 +73,7 @@ void concatPrefixFeature(Array *prefix, Array *feature) { // the recursive feature. Might want to fix it to choose a random number upper // bounded by a static value instead. terminal *featureptr; - int len = rand() % RECUR_THRESHOLD; + int len = rand_below(global_afl, RECUR_THRESHOLD); for (int x = 0; x < len; x++) { for (int y = 0; y < feature->used; y++) { @@ -149,7 +149,7 @@ Array *gen_input(state *pda, Array *input) { state_ptr = pda + curr_state; // Get a random trigger - randval = rand() % (state_ptr->trigger_len); + randval = rand_below(global_afl, state_ptr->trigger_len); trigger_ptr = (state_ptr->ptr) + randval; // Insert into the dynamic array @@ -187,7 +187,7 @@ Array *gen_input_count(state *pda, Array *input, int *mut_count) { state_ptr = pda + curr_state; // Get a random trigger - randval = rand() % (state_ptr->trigger_len); + randval = rand_below(global_afl, state_ptr->trigger_len); trigger_ptr = (state_ptr->ptr) + randval; // Insert into the dynamic array diff --git a/custom_mutators/gramatron/gramfuzz-mutators.c b/custom_mutators/gramatron/gramfuzz-mutators.c index 0255e1d0..0fc9c307 100644 --- a/custom_mutators/gramatron/gramfuzz-mutators.c +++ b/custom_mutators/gramatron/gramfuzz-mutators.c @@ -13,7 +13,7 @@ Array *performRandomMutation(state *pda, Array *input) { Array *sliced; // Get offset at which to generate new input and slice it - int idx = rand() % input->used; + int idx = rand_below(global_afl, input->used); sliced = slice(input, idx); // print_repr(sliced, "Slice"); @@ -58,7 +58,7 @@ Array *performSpliceOne(Array *originput, IdxMap_new *statemap_orig, int length = utarray_len(stateptr); if (length) { - int *splice_idx = (int *)utarray_eltptr(stateptr, rand() % length); + int *splice_idx = (int *)utarray_eltptr(stateptr, rand_below(global_afl, length)); ip.orig_idx = *splice_idx; ip.splice_idx = x; utarray_push_back(pairs, &ip); @@ -69,7 +69,7 @@ Array *performSpliceOne(Array *originput, IdxMap_new *statemap_orig, // Pick a random pair int length = utarray_len(pairs); - cand = (intpair_t *)utarray_eltptr(pairs, rand() % length); + cand = (intpair_t *)utarray_eltptr(pairs, rand_below(global_afl, length)); // printf("\n Orig_idx:%d Splice_idx:%d", cand->orig_idx, cand->splice_idx); // Perform the splicing @@ -162,7 +162,7 @@ UT_array **get_dupes(Array *input, int *recur_len) { Array *doMult(Array *input, UT_array **recur, int recurlen) { int offset = 0; - int idx = rand() % (recurlen); + int idx = rand_below(global_afl, recurlen); UT_array *recurMap = recur[idx]; UT_array *recurPtr; Array * prefix; @@ -225,14 +225,13 @@ void getTwoIndices(UT_array *recur, int recurlen, int *firstIdx, for (int i = offset - 1; i > 0; i--) { // Pick a random index from 0 to i - int j = rand() % (i + 1); + int j = rand_below(global_afl, i + 1); // Swap arr[i] with the element at random index swap(&ArrayRecurIndices[i], &ArrayRecurIndices[j]); } - // Get the first two indices *firstIdx = ArrayRecurIndices[0]; *secondIdx = ArrayRecurIndices[1]; diff --git a/custom_mutators/gramatron/gramfuzz.c b/custom_mutators/gramatron/gramfuzz.c index fd126ec0..d64d2fa9 100644 --- a/custom_mutators/gramatron/gramfuzz.c +++ b/custom_mutators/gramatron/gramfuzz.c @@ -125,7 +125,6 @@ state *create_pda(u8 *automaton_file) { my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { - srand(seed); my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); if (!data) { @@ -142,6 +141,7 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { } data->afl = afl; + global_afl = afl; // dirty data->seed = seed; data->mut_alloced = 0; @@ -211,7 +211,7 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, } else if (data->mut_idx == 2) { // Perform splice mutation // we cannot use the supplied splice data so choose a new random file - u32 tid = rand() % data->afl->queued_paths; + u32 tid = rand_below(global_afl, data->afl->queued_paths); struct queue_entry *q = data->afl->queue_buf[tid]; // Read the input representation for the splice candidate diff --git a/custom_mutators/gramatron/gramfuzz.h b/custom_mutators/gramatron/gramfuzz.h index e6912074..1e599f0c 100644 --- a/custom_mutators/gramatron/gramfuzz.h +++ b/custom_mutators/gramatron/gramfuzz.h @@ -23,6 +23,8 @@ 3600 // Inputs that gave new coverage will be dumped every FLUSH_INTERVAL // seconds +afl_state_t *global_afl; + typedef struct trigger { char * id; diff --git a/custom_mutators/gramatron/json-c b/custom_mutators/gramatron/json-c new file mode 160000 index 00000000..af8dd4a3 --- /dev/null +++ b/custom_mutators/gramatron/json-c @@ -0,0 +1 @@ +Subproject commit af8dd4a307e7b837f9fa2959549548ace4afe08b diff --git a/docs/Changelog.md b/docs/Changelog.md index 3f3dc642..cb22c272 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -9,7 +9,7 @@ Want to stay in the loop on major new features? Join our mailing list by sending a mail to . ### Version ++3.15a (dev) - - ... + - added the very good grammar mutator "GramaTron" to the custom_mutators ### Version ++3.14c (release) -- cgit 1.4.1