From e1d5009229fb5cea5845cd08e0abdc8fe440ee86 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 9 Jul 2021 10:32:14 +0200 Subject: fixes --- custom_mutators/gramatron/gramfuzz-helpers.c | 336 +++++++++++++++++++++++++++ 1 file changed, 336 insertions(+) create mode 100644 custom_mutators/gramatron/gramfuzz-helpers.c (limited to 'custom_mutators/gramatron/gramfuzz-helpers.c') diff --git a/custom_mutators/gramatron/gramfuzz-helpers.c b/custom_mutators/gramatron/gramfuzz-helpers.c new file mode 100644 index 00000000..f894c850 --- /dev/null +++ b/custom_mutators/gramatron/gramfuzz-helpers.c @@ -0,0 +1,336 @@ +#include +#include +#include +#include +#include "afl-fuzz.h" +#include "gramfuzz.h" + +/*Slices from beginning till idx*/ +Array *slice(Array *input, int idx) { + + // printf("\nSlice idx:%d", idx); + terminal *origptr; + terminal *term_ptr; + Array * sliced = (Array *)malloc(sizeof(Array)); + initArray(sliced, input->size); + // Populate dynamic array members + if (idx == 0) { return sliced; } + for (int x = 0; x < idx; x++) { + + origptr = &input->start[x]; + insertArray(sliced, origptr->state, origptr->symbol, origptr->symbol_len, + origptr->trigger_idx); + + } + + return sliced; + +} + +/* Slices from idx till end*/ +Array *slice_inverse(Array *input, int idx) { + + // printf("\nSlice idx:%d", idx); + terminal *origptr; + terminal *term_ptr; + Array * sliced = (Array *)malloc(sizeof(Array)); + initArray(sliced, input->size); + for (int x = idx; x < input->used; x++) { + + origptr = &input->start[x]; + insertArray(sliced, origptr->state, origptr->symbol, origptr->symbol_len, + origptr->trigger_idx); + + } + + return sliced; + +} + +/*Carves with `start` included and `end` excluded*/ +Array *carve(Array *input, int start, int end) { + + terminal *origptr; + terminal *term_ptr; + Array * sliced = (Array *)malloc(sizeof(Array)); + initArray(sliced, input->size); + for (int x = start; x < end; x++) { + + origptr = &input->start[x]; + insertArray(sliced, origptr->state, origptr->symbol, origptr->symbol_len, + origptr->trigger_idx); + + } + + return sliced; + +} + +/*Concats prefix + feature *mult*/ +void concatPrefixFeature(Array *prefix, Array *feature) { + + // XXX: Currently we have hardcoded the multiplication threshold for adding + // the recursive feature. Might want to fix it to choose a random number upper + // bounded by a static value instead. + terminal *featureptr; + int len = rand() % RECUR_THRESHOLD; + for (int x = 0; x < len; x++) { + + for (int y = 0; y < feature->used; y++) { + + featureptr = &feature->start[y]; + insertArray(prefix, featureptr->state, featureptr->symbol, + featureptr->symbol_len, featureptr->trigger_idx); + + } + + } + +} + +void concatPrefixFeatureBench(Array *prefix, Array *feature) { + + // XXX: Currently we have hardcoded the multiplication threshold for adding + // the recursive feature. Might want to fix it to choose a random number upper + // bounded by a static value instead. + terminal *featureptr; + int len = + 5; // 5 is the number of times we compare performing random recursion. + for (int x = 0; x < len; x++) { + + for (int y = 0; y < feature->used; y++) { + + featureptr = &feature->start[y]; + insertArray(prefix, featureptr->state, featureptr->symbol, + featureptr->symbol_len, featureptr->trigger_idx); + + } + + } + +} + +Array *spliceGF(Array *orig, Array *toSplice, int idx) { + + terminal *toSplicePtr; + terminal *tempPtr; + // Iterate through the splice candidate from the `idx` till end + for (int x = idx; x < toSplice->used; x++) { + + toSplicePtr = &toSplice->start[x]; + insertArray(orig, toSplicePtr->state, toSplicePtr->symbol, + toSplicePtr->symbol_len, toSplicePtr->trigger_idx); + + } + + return orig; + +} + +Array *gen_input(state *pda, Array *input) { + + state * state_ptr; + trigger * trigger_ptr; + terminal *term_ptr; + int offset = 0; + int randval, error; + // Generating an input for the first time + if (input == NULL) { + + input = (Array *)calloc(1, sizeof(Array)); + initArray(input, INIT_SIZE); + curr_state = init_state; + + } + + while (curr_state != final_state) { + + // Retrieving the state from the pda + state_ptr = pda + curr_state; + + // Get a random trigger + randval = rand() % (state_ptr->trigger_len); + trigger_ptr = (state_ptr->ptr) + randval; + + // Insert into the dynamic array + insertArray(input, curr_state, trigger_ptr->term, trigger_ptr->term_len, + randval); + curr_state = trigger_ptr->dest; + offset += 1; + + } + + return input; + +} + +Array *gen_input_count(state *pda, Array *input, int *mut_count) { + + state * state_ptr; + trigger * trigger_ptr; + terminal *term_ptr; + int offset = 0; + int randval, error; + // Generating an input for the first time + if (input == NULL) { + + input = (Array *)calloc(1, sizeof(Array)); + initArray(input, INIT_SIZE); + curr_state = init_state; + + } + + while (curr_state != final_state) { + + *mut_count += 1; + // Retrieving the state from the pda + state_ptr = pda + curr_state; + + // Get a random trigger + randval = rand() % (state_ptr->trigger_len); + trigger_ptr = (state_ptr->ptr) + randval; + + // Insert into the dynamic array + insertArray(input, curr_state, trigger_ptr->term, trigger_ptr->term_len, + randval); + curr_state = trigger_ptr->dest; + offset += 1; + + } + + return input; + +} + +/*Creates a candidate from walk with state hashmap and + * recursion hashmap + */ + +Candidate *gen_candidate(Array *input) { + + terminal * term_ptr; + IdxMap_new *idxmapPtr; + // Declare the State Hash Table + IdxMap_new *idxmapStart = + (IdxMap_new *)malloc(sizeof(IdxMap_new) * numstates); + for (int x = 0; x < numstates; x++) { + + idxmapPtr = &idxmapStart[x]; + utarray_new(idxmapPtr->nums, &ut_int_icd); + + } + + char * trigger; + int state; + char * key; + Candidate *candidate = (Candidate *)malloc(sizeof(Candidate)); + candidate->walk = input; + int offset = 0, error; + + // Generate statemap for splicing + while (offset < input->used) { + + term_ptr = &input->start[offset]; + state = term_ptr->state; + // char *statenum = state + 1; + // int num = atoi(statenum); + idxmapPtr = &idxmapStart[state]; + utarray_push_back(idxmapPtr->nums, &offset); + offset += 1; + + } + + candidate->statemap = idxmapStart; + return candidate; + +} + +char *get_state(char *trigger) { + + // Get the state from transition + int trigger_idx = 0; + printf("\nTrigger:%s", trigger); + char *state = (char *)malloc(sizeof(char) * 10); + while (trigger[trigger_idx] != '_') { + + state[trigger_idx] = trigger[trigger_idx]; + trigger_idx += 1; + + } + + printf("\nTrigger Idx:%d", trigger_idx); + state[trigger_idx] = '\0'; + return state; + +} + +void print_repr(Array *input, char *prefix) { + + size_t offset = 0; + terminal *term_ptr; + char geninput[input->used * 100]; + if (!input->used) { + + printf("\n============="); + printf("\n%s:%s", prefix, ""); + printf("\n============="); + return; + + } + + // This is done to create a null-terminated initial string + term_ptr = &input->start[offset]; + strcpy(geninput, term_ptr->symbol); + offset += 1; + + while (offset < input->used) { + + term_ptr = &input->start[offset]; + strcat(geninput, term_ptr->symbol); + offset += 1; + + } + + printf("\n============="); + printf("\n%s:%s", prefix, geninput); + printf("\n============="); + +} + +// int main(int argc, char*argv[]) { + +// char *mode; +// if (argc == 1) { + +// printf("\nUsage: ./gramfuzzer "); +// return -1; +// } +// if (argc >= 2) { + +// mode = argv[1]; +// printf("\nMode:%s", mode); +// } +// if (! strcmp(mode, "Generate")) { + +// GenInputBenchmark(); +// } +// else if (! strcmp(mode, "RandomMutation")) { + +// RandomMutationBenchmark(); +// } +// else if (! strcmp(mode, "Splice")) { + +// SpliceMutationBenchmark(); +// } +// else if (! strcmp(mode, "Recursive")) { + +// RandomRecursiveBenchmark(); +// } +// else { + +// printf("\nUnrecognized mode"); +// return -1; +// } +// return 0; +// } + -- cgit 1.4.1 From 3101e9c88df72755b461e82870879fbe9e7429fa Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 20 Jul 2021 11:10:58 +0200 Subject: add gramatron --- .gitmodules | 3 + custom_mutators/gramatron/JSONC_VERSION | 1 + custom_mutators/gramatron/Makefile | 7 -- custom_mutators/gramatron/README.md | 18 +-- .../gramatron/build_gramatron_mutator.sh | 140 +++++++++++++++++++++ custom_mutators/gramatron/gramfuzz-helpers.c | 6 +- custom_mutators/gramatron/gramfuzz-mutators.c | 11 +- custom_mutators/gramatron/gramfuzz.c | 4 +- custom_mutators/gramatron/gramfuzz.h | 2 + custom_mutators/gramatron/json-c | 1 + docs/Changelog.md | 2 +- 11 files changed, 164 insertions(+), 31 deletions(-) create mode 100644 custom_mutators/gramatron/JSONC_VERSION delete mode 100644 custom_mutators/gramatron/Makefile create mode 100755 custom_mutators/gramatron/build_gramatron_mutator.sh create mode 160000 custom_mutators/gramatron/json-c (limited to 'custom_mutators/gramatron/gramfuzz-helpers.c') diff --git a/.gitmodules b/.gitmodules index c787ec0e..6edefd72 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "qemu_mode/qemuafl"] path = qemu_mode/qemuafl url = https://github.com/AFLplusplus/qemuafl +[submodule "custom_mutators/gramatron/json-c"] + path = custom_mutators/gramatron/json-c + url = https://github.com/json-c/json-c diff --git a/custom_mutators/gramatron/JSONC_VERSION b/custom_mutators/gramatron/JSONC_VERSION new file mode 100644 index 00000000..7663833a --- /dev/null +++ b/custom_mutators/gramatron/JSONC_VERSION @@ -0,0 +1 @@ +af8dd4a307e7b837f9fa2959549548ace4afe08b diff --git a/custom_mutators/gramatron/Makefile b/custom_mutators/gramatron/Makefile deleted file mode 100644 index d24f3dd4..00000000 --- a/custom_mutators/gramatron/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -all: gramatron.so - -gramatron.so: gramfuzz.c gramfuzz.h gramfuzz-helpers.c gramfuzz-mutators.c gramfuzz-util.c hashmap.c hashmap.h test.c test.h utarray.h uthash.h - $(CC) -O3 -g -fPIC -Wno-unused-result -Wl,--allow-multiple-definition -I../../include -o gramatron.so -shared -I. -I/prg/dev/include gramfuzz.c gramfuzz-helpers.c gramfuzz-mutators.c gramfuzz-util.c hashmap.c test.c -ljson-c - -clean: - rm -f gramatron.so diff --git a/custom_mutators/gramatron/README.md b/custom_mutators/gramatron/README.md index 6659cb95..91f93355 100644 --- a/custom_mutators/gramatron/README.md +++ b/custom_mutators/gramatron/README.md @@ -1,20 +1,15 @@ # GramaTron Gramatron is a coverage-guided fuzzer that uses grammar automatons to perform -grammar-aware fuzzing. Technical details about our framework are available in our -[ISSTA'21 paper](https://nebelwelt.net/files/21ISSTA.pdf). The artifact to reproduce the -experiments presented in our paper are present in `artifact/`. Instructions to run -a sample campaign and incorporate new grammars is presented below: +grammar-aware fuzzing. Technical details about our framework are available +in the [ISSTA'21 paper](https://nebelwelt.net/files/21ISSTA.pdf). +The artifact to reproduce the experiments presented in the paper are present +in `artifact/`. Instructions to run a sample campaign and incorporate new +grammars is presented below: # Compiling -- Install `json-c` -``` -git clone https://github.com/json-c/json-c.git -cd json-c && git reset --hard af8dd4a307e7b837f9fa2959549548ace4afe08b && sh autogen.sh && ./configure && make && make install -``` - -afterwards you can just `make` GrammaTron +Simply execute `./build_gramatron_mutator.sh` # Running @@ -48,4 +43,3 @@ Eg. ./preprocess/prep_automaton.sh ~/grammars/ruby/source.json PROGRAM Eg. ./test SanityCheck ~/grammars/ruby/source_automata.json ``` - diff --git a/custom_mutators/gramatron/build_gramatron_mutator.sh b/custom_mutators/gramatron/build_gramatron_mutator.sh new file mode 100755 index 00000000..c1cdf0e7 --- /dev/null +++ b/custom_mutators/gramatron/build_gramatron_mutator.sh @@ -0,0 +1,140 @@ +#!/bin/sh +# +# american fuzzy lop++ - gramatron build script +# ------------------------------------------------ +# +# Originally written by Nathan Voss +# +# Adapted from code by Andrew Griffiths and +# Michal Zalewski +# +# Adapted for AFLplusplus by Dominik Maier +# +# Copyright 2017 Battelle Memorial Institute. All rights reserved. +# Copyright 2019-2020 AFLplusplus Project. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# This script downloads, patches, and builds a version of Unicorn with +# minor tweaks to allow Unicorn-emulated binaries to be run under +# afl-fuzz. +# +# The modifications reside in patches/*. The standalone Unicorn library +# will be written to /usr/lib/libunicornafl.so, and the Python bindings +# will be installed system-wide. +# +# You must make sure that Unicorn Engine is not already installed before +# running this script. If it is, please uninstall it first. + +JSONC_VERSION="$(cat ./JSONC_VERSION)" +JSONC_REPO="https://github.com/json-c/json-c" + +echo "=================================================" +echo "Gramatron Mutator build script" +echo "=================================================" +echo + +echo "[*] Performing basic sanity checks..." + +PLT=`uname -s` + +if [ ! -f "../../config.h" ]; then + + echo "[-] Error: key files not found - wrong working directory?" + exit 1 + +fi + +PYTHONBIN=`command -v python3 || command -v python || command -v python2 || echo python3` +MAKECMD=make +TARCMD=tar + +if [ "$PLT" = "Darwin" ]; then + CORES=`sysctl -n hw.ncpu` + TARCMD=tar +fi + +if [ "$PLT" = "FreeBSD" ]; then + MAKECMD=gmake + CORES=`sysctl -n hw.ncpu` + TARCMD=gtar +fi + +if [ "$PLT" = "NetBSD" ] || [ "$PLT" = "OpenBSD" ]; then + MAKECMD=gmake + CORES=`sysctl -n hw.ncpu` + TARCMD=gtar +fi + +PREREQ_NOTFOUND= +for i in git $MAKECMD $TARCMD; do + + T=`command -v "$i" 2>/dev/null` + + if [ "$T" = "" ]; then + + echo "[-] Error: '$i' not found. Run 'sudo apt-get install $i' or similar." + PREREQ_NOTFOUND=1 + + fi + +done + +test -z "$CC" && export CC=cc + +if echo "$CC" | grep -qF /afl-; then + + echo "[-] Error: do not use afl-gcc or afl-clang to compile this tool." + PREREQ_NOTFOUND=1 + +fi + +if [ "$PREREQ_NOTFOUND" = "1" ]; then + exit 1 +fi + +echo "[+] All checks passed!" + +echo "[*] Making sure json-c is checked out" + +git status 1>/dev/null 2>/dev/null +if [ $? -eq 0 ]; then + echo "[*] initializing json-c submodule" + git submodule init || exit 1 + git submodule update ./json-c 2>/dev/null # ignore errors +else + echo "[*] cloning json-c" + test -d json-c || { + CNT=1 + while [ '!' -d json-c -a "$CNT" -lt 4 ]; do + echo "Trying to clone json-c (attempt $CNT/3)" + git clone "$JSONC_REPO" + CNT=`expr "$CNT" + 1` + done + } +fi + +test -d json-c || { echo "[-] not checked out, please install git or check your internet connection." ; exit 1 ; } +echo "[+] Got json-c." + +cd "json-c" || exit 1 +echo "[*] Checking out $JSONC_VERSION" +sh -c 'git stash && git stash drop' 1>/dev/null 2>/dev/null +git checkout "$JSONC_VERSION" || exit 1 +sh autogen.sh || exit 1 +export CFLAGS=-fPIC +./configure --disable-shared || exit 1 +make || exit 1 +cd .. + +echo +echo +echo "[+] Json-c successfully prepared!" +echo "[+] Builing gramatron now." +$CC -O3 -g -fPIC -Wno-unused-result -Wl,--allow-multiple-definition -I../../include -o gramatron.so -shared -I. -I/prg/dev/include gramfuzz.c gramfuzz-helpers.c gramfuzz-mutators.c gramfuzz-util.c hashmap.c json-c/.libs/libjson-c.a || exit 1 +echo +echo "[+] gramatron successfully built!" diff --git a/custom_mutators/gramatron/gramfuzz-helpers.c b/custom_mutators/gramatron/gramfuzz-helpers.c index f894c850..378a3d90 100644 --- a/custom_mutators/gramatron/gramfuzz-helpers.c +++ b/custom_mutators/gramatron/gramfuzz-helpers.c @@ -73,7 +73,7 @@ void concatPrefixFeature(Array *prefix, Array *feature) { // the recursive feature. Might want to fix it to choose a random number upper // bounded by a static value instead. terminal *featureptr; - int len = rand() % RECUR_THRESHOLD; + int len = rand_below(global_afl, RECUR_THRESHOLD); for (int x = 0; x < len; x++) { for (int y = 0; y < feature->used; y++) { @@ -149,7 +149,7 @@ Array *gen_input(state *pda, Array *input) { state_ptr = pda + curr_state; // Get a random trigger - randval = rand() % (state_ptr->trigger_len); + randval = rand_below(global_afl, state_ptr->trigger_len); trigger_ptr = (state_ptr->ptr) + randval; // Insert into the dynamic array @@ -187,7 +187,7 @@ Array *gen_input_count(state *pda, Array *input, int *mut_count) { state_ptr = pda + curr_state; // Get a random trigger - randval = rand() % (state_ptr->trigger_len); + randval = rand_below(global_afl, state_ptr->trigger_len); trigger_ptr = (state_ptr->ptr) + randval; // Insert into the dynamic array diff --git a/custom_mutators/gramatron/gramfuzz-mutators.c b/custom_mutators/gramatron/gramfuzz-mutators.c index 0255e1d0..0fc9c307 100644 --- a/custom_mutators/gramatron/gramfuzz-mutators.c +++ b/custom_mutators/gramatron/gramfuzz-mutators.c @@ -13,7 +13,7 @@ Array *performRandomMutation(state *pda, Array *input) { Array *sliced; // Get offset at which to generate new input and slice it - int idx = rand() % input->used; + int idx = rand_below(global_afl, input->used); sliced = slice(input, idx); // print_repr(sliced, "Slice"); @@ -58,7 +58,7 @@ Array *performSpliceOne(Array *originput, IdxMap_new *statemap_orig, int length = utarray_len(stateptr); if (length) { - int *splice_idx = (int *)utarray_eltptr(stateptr, rand() % length); + int *splice_idx = (int *)utarray_eltptr(stateptr, rand_below(global_afl, length)); ip.orig_idx = *splice_idx; ip.splice_idx = x; utarray_push_back(pairs, &ip); @@ -69,7 +69,7 @@ Array *performSpliceOne(Array *originput, IdxMap_new *statemap_orig, // Pick a random pair int length = utarray_len(pairs); - cand = (intpair_t *)utarray_eltptr(pairs, rand() % length); + cand = (intpair_t *)utarray_eltptr(pairs, rand_below(global_afl, length)); // printf("\n Orig_idx:%d Splice_idx:%d", cand->orig_idx, cand->splice_idx); // Perform the splicing @@ -162,7 +162,7 @@ UT_array **get_dupes(Array *input, int *recur_len) { Array *doMult(Array *input, UT_array **recur, int recurlen) { int offset = 0; - int idx = rand() % (recurlen); + int idx = rand_below(global_afl, recurlen); UT_array *recurMap = recur[idx]; UT_array *recurPtr; Array * prefix; @@ -225,14 +225,13 @@ void getTwoIndices(UT_array *recur, int recurlen, int *firstIdx, for (int i = offset - 1; i > 0; i--) { // Pick a random index from 0 to i - int j = rand() % (i + 1); + int j = rand_below(global_afl, i + 1); // Swap arr[i] with the element at random index swap(&ArrayRecurIndices[i], &ArrayRecurIndices[j]); } - // Get the first two indices *firstIdx = ArrayRecurIndices[0]; *secondIdx = ArrayRecurIndices[1]; diff --git a/custom_mutators/gramatron/gramfuzz.c b/custom_mutators/gramatron/gramfuzz.c index fd126ec0..d64d2fa9 100644 --- a/custom_mutators/gramatron/gramfuzz.c +++ b/custom_mutators/gramatron/gramfuzz.c @@ -125,7 +125,6 @@ state *create_pda(u8 *automaton_file) { my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { - srand(seed); my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); if (!data) { @@ -142,6 +141,7 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { } data->afl = afl; + global_afl = afl; // dirty data->seed = seed; data->mut_alloced = 0; @@ -211,7 +211,7 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, } else if (data->mut_idx == 2) { // Perform splice mutation // we cannot use the supplied splice data so choose a new random file - u32 tid = rand() % data->afl->queued_paths; + u32 tid = rand_below(global_afl, data->afl->queued_paths); struct queue_entry *q = data->afl->queue_buf[tid]; // Read the input representation for the splice candidate diff --git a/custom_mutators/gramatron/gramfuzz.h b/custom_mutators/gramatron/gramfuzz.h index e6912074..1e599f0c 100644 --- a/custom_mutators/gramatron/gramfuzz.h +++ b/custom_mutators/gramatron/gramfuzz.h @@ -23,6 +23,8 @@ 3600 // Inputs that gave new coverage will be dumped every FLUSH_INTERVAL // seconds +afl_state_t *global_afl; + typedef struct trigger { char * id; diff --git a/custom_mutators/gramatron/json-c b/custom_mutators/gramatron/json-c new file mode 160000 index 00000000..af8dd4a3 --- /dev/null +++ b/custom_mutators/gramatron/json-c @@ -0,0 +1 @@ +Subproject commit af8dd4a307e7b837f9fa2959549548ace4afe08b diff --git a/docs/Changelog.md b/docs/Changelog.md index 3f3dc642..cb22c272 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -9,7 +9,7 @@ Want to stay in the loop on major new features? Join our mailing list by sending a mail to . ### Version ++3.15a (dev) - - ... + - added the very good grammar mutator "GramaTron" to the custom_mutators ### Version ++3.14c (release) -- cgit 1.4.1