diff options
24 files changed, 1189 insertions, 803 deletions
diff --git a/.gitignore b/.gitignore index 4032b826..bdd342c4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,9 +18,11 @@ afl-qemu-trace afl-showmap afl-tmin afl-analyze.8 +afl-as.8 afl-clang-fast++.8 afl-clang-fast.8 afl-cmin.8 +afl-cmin.bash.8 afl-fuzz.8 afl-gcc.8 afl-gcc-fast.8 diff --git a/docs/custom_mutator.md b/docs/custom_mutator.md deleted file mode 100644 index dff32c1d..00000000 --- a/docs/custom_mutator.md +++ /dev/null @@ -1,45 +0,0 @@ -# Adding custom mutators to AFL - -This file describes how you can implement custom mutations to be used in AFL. - -Implemented by Khaled Yakdan from Code Intelligence <yakdan@code-intelligence.de> - -## 1) Description - -Custom mutator libraries can be passed to afl-fuzz to perform custom mutations -on test cases beyond those available in AFL - for example, to enable -structure-aware fuzzing by using libraries that perform mutations according to -a given grammar. - -The custom mutator library is passed to afl-fuzz via the -AFL_CUSTOM_MUTATOR_LIBRARY environment variable. The library must export -the afl_custom_mutator() function and must be compiled as a shared object. -For example: -``` -$CC -shared -Wall -O3 <lib-name>.c -o <lib-name>.so -``` -Note: unless AFL_CUSTOM_MUTATOR_ONLY is set, it is a state mutator like any -other, so it will be used for some test cases, and other mutators for others. - -Only if AFL_CUSTOM_MUTATOR_ONLY is set the afl_custom_mutator() function will -be called every time it needs to mutate a test case. - -For some cases, the format of the mutated data returned from the custom -mutator is not suitable to directly execute the target with this input. -For example, when using libprotobuf-mutator, the data returned is in a -protobuf format which corresponds to a given grammar. -In order to execute the target, the protobuf data must be converted to the -plain-text format expected by the target. -In such scenarios, the user can define the afl_pre_save_handler() function. -This function is then transforms the data into the format expected by the -API before executing the target. -afl_pre_save_handler is optional and does not have to be implemented if its -functionality is not needed. - -## 2) Example - -A simple example is provided in ../examples/custom_mutators/ - -There is also a libprotobuf example available at [https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator) -Another implementation can be found at [https://github.com/thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator) - diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md new file mode 100644 index 00000000..4deb07e1 --- /dev/null +++ b/docs/custom_mutators.md @@ -0,0 +1,201 @@ +# Custom Mutators in AFL++ + +This file describes how you can implement custom mutations to be used in AFL. +For now, we support C/C++ library and Python module, collectivelly named as the +custom mutator. + +Implemented by +- C/C++ library (`*.so`): Khaled Yakdan from Code Intelligence (<yakdan@code-intelligence.de>) +- Python module: Christian Holler from Mozilla (<choller@mozilla.com>) + +## 1) Introduction + +Custom mutators can be passed to `afl-fuzz` to perform custom mutations on test +cases beyond those available in AFL. For example, to enable structure-aware +fuzzing by using libraries that perform mutations according to a given grammar. + +The custom mutator is passed to `afl-fuzz` via the `AFL_CUSTOM_MUTATOR_LIBRARY` +or `AFL_PYTHON_MODULE` environment variable., and must export a fuzz function. +Please see [APIs](#2-apis) and [Usage](#3-usage) for detail. + +The custom mutation stage is set to be the first non-deterministic stage (right before the havoc stage). + +Note: If `AFL_CUSTOM_MUTATOR_ONLY` is set, all mutations will solely be +performed with the custom mutator. + +## 2) APIs + +C/C++: +```c +void afl_custom_init(unsigned int seed); +size_t afl_custom_fuzz(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size); +size_t afl_custom_pre_save(u8* buf, size_t buf_size, u8** out_buf); +u32 afl_custom_init_trim(u8* buf, size_t buf_size); +void afl_custom_trim(u8** out_buf, size_t* out_buf_size); +u32 afl_custom_post_trim(u8 success); +``` + +Python: +```python +def init(seed): + pass + +def fuzz(buf, add_buf, max_size): + return mutated_out + +def pre_save(buf): + return out_buf + +def init_trim(buf): + return cnt + +def trim(): + return out_buf + +def post_trim(success): + return next_index +``` + +### Custom Mutation + +- `init` (optional): + + This method is called when AFL++ starts up and is used to seed RNG. + +- `fuzz` (required): + + This method performs custom mutations on a given input. It also accepts an + additional test case. + +- `pre_save` (optional): + + For some cases, the format of the mutated data returned from the custom + mutator is not suitable to directly execute the target with this input. + For example, when using libprotobuf-mutator, the data returned is in a + protobuf format which corresponds to a given grammar. In order to execute + the target, the protobuf data must be converted to the plain-text format expected by the target. In such scenarios, the user can define the + `pre_save` function. This function is then transforms the data into the + format expected by the API before executing the target. + + +### Trimming Support + +The generic trimming routines implemented in AFL++ can easily destroy the +structure of complex formats, possibly leading to a point where you have a lot +of test cases in the queue that your Python module cannot process anymore but +your target application still accepts. This is especially the case when your +target can process a part of the input (causing coverage) and then errors out +on the remaining input. + +In such cases, it makes sense to implement a custom trimming routine. The API +consists of multiple methods because after each trimming step, we have to go +back into the C code to check if the coverage bitmap is still the same for the +trimmed input. Here's a quick API description: + +- `init_trim` (optional): + + This method is called at the start of each trimming operation and receives + the initial buffer. It should return the amount of iteration steps possible + on this input (e.g. if your input has n elements and you want to remove them + one by one, return n, if you do a binary search, return log(n), and so on). + + If your trimming algorithm doesn't allow you to determine the amount of + (remaining) steps easily (esp. while running), then you can alternatively + return 1 here and always return 0 in `post_trim` until you are finished and + no steps remain. In that case, returning 1 in `post_trim` will end the + trimming routine. The whole current index/max iterations stuff is only used + to show progress. + +- `trim` (optional) + + This method is called for each trimming operation. It doesn't have any + arguments because we already have the initial buffer from `init_trim` and we + can memorize the current state in global variables. This can also save + reparsing steps for each iteration. It should return the trimmed input + buffer, where the returned data must not exceed the initial input data in + length. Returning anything that is larger than the original data (passed to + `init_trim`) will result in a fatal abort of AFL++. + +- `post_trim` (optional) + + This method is called after each trim operation to inform you if your + trimming step was successful or not (in terms of coverage). If you receive + a failure here, you should reset your input to the last known good state. + In any case, this method must return the next trim iteration index (from 0 + to the maximum amount of steps you returned in `init_trim`). + +Omitting any of three methods will cause the trimming to be disabled and trigger +a fallback to the builtin default trimming routine. + +### Environment Variables + +Optionally, the following environment variables are supported: + +- `AFL_PYTHON_ONLY` + + Disable all other mutation stages. This can prevent broken testcases + (those that your Python module can't work with anymore) to fill up your + queue. Best combined with a custom trimming routine (see below) because + trimming can cause the same test breakage like havoc and splice. + +- `AFL_DEBUG` + + When combined with `AFL_NO_UI`, this causes the C trimming code to emit additional messages about the performance and actions of your custom trimmer. Use this to see if it works :) + +## 3) Usage + +### Prerequisite + +For Python mutator, the python 3 or 2 development package is required. On +Debian/Ubuntu/Kali this can be done: + +```bash +sudo apt install python3-dev +# or +sudo apt install python-dev +``` + +Then, AFL++ can be compiled with Python support. The AFL++ Makefile detects +Python 2 and 3 through `python-config` if it is in the PATH and compiles +`afl-fuzz` with the feature if available. + +Note: for some distributions, you might also need the package `python[23]-apt`. +In case your setup is different, set the necessary variables like this: +`PYTHON_INCLUDE=/path/to/python/include LDFLAGS=-L/path/to/python/lib make`. + +### Custom Mutator Preparation + +For C/C++ mutator, the source code must be compiled as a shared object: +```bash +gcc -shared -Wall -O3 example.c -o example.so +``` + +### Run + +C/C++ +```bash +export AFL_CUSTOM_MUTATOR_LIBRARY=/full/path/to/example.so +afl-fuzz /path/to/program +``` + +Python +```bash +export PYTHONPATH=`dirname /full/path/to/example.py` +export AFL_PYTHON_MODULE=example +afl-fuzz /path/to/program +``` + +## 4) Example + +Please see [example.c](../examples/custom_mutators/example.c) and +[example.py](../examples/custom_mutators/example.py) + +## 5) Other Resources + +- AFL libprotobuf mutator + - [bruce30262/libprotobuf-mutator_fuzzing_learning](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator) + - [thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator) +- [XML Fuzzing@NullCon 2017](https://www.agarri.fr/docs/XML_Fuzzing-NullCon2017-PUBLIC.pdf) + - [A bug detected by AFL + XML-aware mutators](https://bugs.chromium.org/p/chromium/issues/detail?id=930663) diff --git a/docs/env_variables.md b/docs/env_variables.md index 527f1c1b..d1cf6977 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -104,7 +104,7 @@ Then there are a few specific features that are only available in llvm_mode: - Setting AFL_LLVM_LAF_SPLIT_COMPARES will split all floating point and 64, 32 and 16 bit integer CMP instructions - See llvm_mode/README.laf-intel.md for more information. + See llvm_mode/README.laf-intel.md for more information. ### WHITELIST @@ -192,7 +192,7 @@ checks or alter some of the more exotic semantics of the tool: deciding if a particular test case is a "hang". The default is 1 second or the value of the -t parameter, whichever is larger. Dialing the value down can be useful if you are very concerned about slow inputs, or if you - don't want AFL to spend too much time classifying that stuff and just + don't want AFL to spend too much time classifying that stuff and just rapidly put all timeouts in that bin. - AFL_NO_ARITH causes AFL to skip most of the deterministic arithmetics. @@ -223,15 +223,15 @@ checks or alter some of the more exotic semantics of the tool: for more. - Setting AFL_CUSTOM_MUTATOR_LIBRARY to a shared library with - afl_custom_mutator() creates additional mutations through this library. + afl_custom_fuzz() creates additional mutations through this library. + If afl-fuzz is compiled with Python (which is autodetected during builing + afl-fuzz), setting AFL_PYTHON_MODULE to a Python module can also provide + additional mutations. If AFL_CUSTOM_MUTATOR_ONLY is also set, all mutations will solely be - performed with/from the library. See [custom_mutator.md](custom_mutator.md) - - - For AFL_PYTHON_MODULE and AFL_PYTHON_ONLY - they require afl-fuzz to - be compiled with Python (which is autodetected during builing afl-fuzz). - Please see [python_mutators.md](python_mutators.md). - This feature allows to configure custom mutators which can be very helpful - in e.g. fuzzing XML or other highly flexible structured input. + performed with the custom mutator. + This feature allows to configure custom mutators which can be very helpful, + e.g. fuzzing XML or other highly flexible structured input. + Please see [custom_mutators.md](custom_mutators.md). - AFL_FAST_CAL keeps the calibration stage about 2.5x faster (albeit less precise), which can help when starting a session against a slow target. @@ -283,7 +283,7 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - Setting AFL_INST_LIBS causes the translator to also instrument the code inside any dynamically linked libraries (notably including glibc). - + - Setting AFL_COMPCOV_LEVEL enables the CompareCoverage tracing of all cmp and sub in x86 and x86_64 and memory comparions functions (e.g. strcmp, memcmp, ...) when libcompcov is preloaded using AFL_PRELOAD. @@ -292,7 +292,7 @@ The QEMU wrapper used to instrument binary-only code supports several settings: only comparisons with immediate values / read-only memory and AFL_COMPCOV_LEVEL=2 that instruments all the comparions. Level 2 is more accurate but may need a larger shared memory. - + - Setting AFL_QEMU_COMPCOV enables the CompareCoverage tracing of all cmp and sub in x86 and x86_64. This is an alias of AFL_COMPCOV_LEVEL=1 when AFL_COMPCOV_LEVEL is @@ -304,25 +304,25 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - AFL_DEBUG will print the found entrypoint for the binary to stderr. Use this if you are unsure if the entrypoint might be wrong - but - use it directly, e.g. afl-qemu-trace ./program + use it directly, e.g. afl-qemu-trace ./program - AFL_ENTRYPOINT allows you to specify a specific entrypoint into the binary (this can be very good for the performance!). The entrypoint is specified as hex address, e.g. 0x4004110 Note that the address must be the address of a basic block. - + - When the target is i386/x86_64 you can specify the address of the function that has to be the body of the persistent loop using AFL_QEMU_PERSISTENT_ADDR=`start addr`. - + - Another modality to execute the persistent loop is to specify also the AFL_QEMU_PERSISTENT_RET=`end addr` env variable. With this variable assigned, instead of patching the return address, the specified instruction is transformed to a jump towards `start addr`. - + - AFL_QEMU_PERSISTENT_GPR=1 QEMU will save the original value of general purpose registers and restore them in each persistent cycle. - + - With AFL_QEMU_PERSISTENT_RETADDR_OFFSET you can specify the offset from the stack pointer in which QEMU can find the return address when `start addr` is hitted. @@ -376,7 +376,7 @@ The library honors these environmental variables: - AFL_LD_NO_CALLOC_OVER inhibits abort() on calloc() overflows. Most of the common allocators check for that internally and return NULL, so it's a security risk only in more exotic setups. - + - AFL_ALIGNED_ALLOC=1 will force the alignment of the allocation size to max_align_t to be compliant with the C standard. @@ -410,7 +410,7 @@ optimal values if not already present in the environment: - In the same vein, by default, MSAN_OPTIONS are set to: - exit_code=86 (required for legacy reasons) + exit_code=86 (required for legacy reasons) abort_on_error=1 symbolize=0 msan_track_origins=0 diff --git a/docs/python_mutators.md b/docs/python_mutators.md deleted file mode 100644 index a7e2c7de..00000000 --- a/docs/python_mutators.md +++ /dev/null @@ -1,148 +0,0 @@ -# Adding custom mutators to AFL using Python modules - - This file describes how you can utilize the external Python API to write - your own custom mutation routines. - - Note: This feature is highly experimental. Use at your own risk. - - Implemented by Christian Holler (:decoder) <choller@mozilla.com>. - - NOTE: Only cPython 2.7, 3.7 and above are supported, although others may work. - Depending on with which version afl-fuzz was compiled against, you must use - python2 or python3 syntax in your scripts! - After a major version upgrade (e.g. 3.7 -> 3.8), a recompilation of afl-fuzz may be needed. - - For an example and a template see ../examples/python_mutators/ - - -## 1) Description and purpose - -While AFLFuzz comes with a good selection of generic deterministic and -non-deterministic mutation operations, it sometimes might make sense to extend -these to implement strategies more specific to the target you are fuzzing. - -For simplicity and in order to allow people without C knowledge to extend -AFLFuzz, I implemented a "Python" stage that can make use of an external -module (written in Python) that implements a custom mutation stage. - -The main motivation behind this is to lower the barrier for people -experimenting with this tool. Hopefully, someone will be able to do useful -things with this extension. - -If you find it useful, have questions or need additional features added to the -interface, feel free to send a mail to <choller@mozilla.com>. - -See the following information to get a better pictures: - https://www.agarri.fr/docs/XML_Fuzzing-NullCon2017-PUBLIC.pdf - https://bugs.chromium.org/p/chromium/issues/detail?id=930663 - - -## 2) How the Python module looks like - -You can find a simple example in pymodules/example.py including documentation -explaining each function. In the same directory, you can find another simple -module that performs simple mutations. - -Right now, "init" is called at program startup and can be used to perform any -kinds of one-time initializations while "fuzz" is called each time a mutation -is requested. - -There is also optional support for a trimming API, see the section below for -further information about this feature. - - -## 3) How to compile AFLFuzz with Python support - -You must install the python 3 or 2 development package of your Linux -distribution before this will work. On Debian/Ubuntu/Kali this can be done -with either: - apt install python3-dev -or - apt install python-dev -Note that for some distributions you might also need the package python[23]-apt - -A prerequisite for using this mode is to compile AFLFuzz with Python support. - -The AFL++ Makefile detects Python 3 and 2 through `python-config` if is is in the PATH -and compiles afl-fuzz with the feature if available. - -In case your setup is different set the necessary variables like this: -PYTHON_INCLUDE=/path/to/python/include LDFLAGS=-L/path/to/python/lib make - - -## 4) How to run AFLFuzz with your custom module - -You must pass the module name inside the env variable AFL_PYTHON_MODULE. - -In addition, if you are trying to load the module from the local directory, -you must adjust your PYTHONPATH to reflect this circumstance. The following -command should work if you are inside the aflfuzz directory: - -$ AFL_PYTHON_MODULE="pymodules.test" PYTHONPATH=. ./afl-fuzz - -Optionally, the following environment variables are supported: - -AFL_PYTHON_ONLY - Disable all other mutation stages. This can prevent broken - testcases (those that your Python module can't work with - anymore) to fill up your queue. Best combined with a custom - trimming routine (see below) because trimming can cause the - same test breakage like havoc and splice. - -AFL_DEBUG - When combined with AFL_NO_UI, this causes the C trimming code - to emit additional messages about the performance and actions - of your custom Python trimmer. Use this to see if it works :) - - -## 5) Order and statistics - -The Python stage is set to be the first non-deterministic stage (right before -the havoc stage). In the statistics however, it shows up as the third number -under "havoc". That's because I'm lazy and I didn't want to mess with the UI -too much ;) - - -## 6) Trimming support - -The generic trimming routines implemented in AFLFuzz can easily destroy the -structure of complex formats, possibly leading to a point where you have a lot -of testcases in the queue that your Python module cannot process anymore but -your target application still accepts. This is especially the case when your -target can process a part of the input (causing coverage) and then errors out -on the remaining input. - -In such cases, it makes sense to implement a custom trimming routine in Python. -The API consists of multiple methods because after each trimming step, we have -to go back into the C code to check if the coverage bitmap is still the same -for the trimmed input. Here's a quick API description: - -init_trim: This method is called at the start of each trimming operation - and receives the initial buffer. It should return the amount - of iteration steps possible on this input (e.g. if your input - has n elements and you want to remove them one by one, return n, - if you do a binary search, return log(n), and so on...). - - If your trimming algorithm doesn't allow you to determine the - amount of (remaining) steps easily (esp. while running), then you - can alternatively return 1 here and always return 0 in post_trim - until you are finished and no steps remain. In that case, - returning 1 in post_trim will end the trimming routine. The whole - current index/max iterations stuff is only used to show progress. - -trim: This method is called for each trimming operation. It doesn't - have any arguments because we already have the initial buffer - from init_trim and we can memorize the current state in global - variables. This can also save reparsing steps for each iteration. - It should return the trimmed input buffer, where the returned data - must not exceed the initial input data in length. Returning anything - that is larger than the original data (passed to init_trim) will - result in a fatal abort of AFLFuzz. - -post_trim: This method is called after each trim operation to inform you - if your trimming step was successful or not (in terms of coverage). - If you receive a failure here, you should reset your input to the - last known good state. - In any case, this method must return the next trim iteration index - (from 0 to the maximum amount of steps you returned in init_trim). - -Omitting any of the methods will cause Python trimming to be disabled and -trigger a fallback to the builtin default trimming routine. diff --git a/examples/custom_mutators/README.md b/examples/custom_mutators/README.md index 6da288ab..ce49436e 100644 --- a/examples/custom_mutators/README.md +++ b/examples/custom_mutators/README.md @@ -1,4 +1,22 @@ -# A simple example for AFL_CUSTOM_MUTATOR_LIBRARY +# Examples for the custom mutator -This is a simple example for the AFL_CUSTOM_MUTATOR_LIBRARY feature. -For more information see [docs/custom_mutator.md](../docs/custom_mutator.md) +These are example and helper files for the custom mutator feature. +See [docs/python_mutators.md](../docs/custom_mutators.md) for more information + +Note that if you compile with python3.7 you must use python3 scripts, and if +you use pyton2.7 to compile python2 scripts! + +example.c - this is a simple example written in C and should be compiled to a + shared library + +example.py - this is the template you can use, the functions are there but they + are empty + +simple-chunk-replace.py - this is a simple example where chunks are replaced + +common.py - this can be used for common functions and helpers. + the examples do not use this though. But you can :) + +wrapper_afl_min.py - mutation of XML documents, loads XmlMutatorMin.py + +XmlMutatorMin.py - module for XML mutation diff --git a/examples/python_mutators/XmlMutatorMin.py b/examples/custom_mutators/XmlMutatorMin.py index 058b7e61..4c80a2ba 100644 --- a/examples/python_mutators/XmlMutatorMin.py +++ b/examples/custom_mutators/XmlMutatorMin.py @@ -7,6 +7,7 @@ from copy import deepcopy from lxml import etree as ET import random, re, io + ########################### # The XmlMutatorMin class # ########################### @@ -40,19 +41,19 @@ class XmlMutatorMin: self.tree = None # High-level mutators (no database needed) - hl_mutators_delete = [ "del_node_and_children", "del_node_but_children", "del_attribute", "del_content" ] # Delete items - hl_mutators_fuzz = ["fuzz_attribute"] # Randomly change attribute values + hl_mutators_delete = ["del_node_and_children", "del_node_but_children", "del_attribute", "del_content"] # Delete items + hl_mutators_fuzz = ["fuzz_attribute"] # Randomly change attribute values # Exposed mutators self.hl_mutators_all = hl_mutators_fuzz + hl_mutators_delete - - def __parse_xml (self, xml): + + def __parse_xml(self, xml): """ Parse an XML string. Basic wrapper around lxml.parse() """ try: # Function parse() takes care of comments / DTD / processing instructions / ... - tree = ET.parse(io.BytesIO(xml)) + tree = ET.parse(io.BytesIO(xml)) except ET.ParseError: raise RuntimeError("XML isn't well-formed!") except LookupError as e: @@ -61,34 +62,34 @@ class XmlMutatorMin: # Return a document wrapper return tree - def __exec_among (self, module, functions, min_times, max_times): + def __exec_among(self, module, functions, min_times, max_times): """ Randomly execute $functions between $min and $max times """ - for i in xrange (random.randint (min_times, max_times)): + for i in xrange(random.randint(min_times, max_times)): # Function names are mangled because they are "private" - getattr (module, "_XmlMutatorMin__" + random.choice(functions)) () + getattr(module, "_XmlMutatorMin__" + random.choice(functions))() - def __serialize_xml (self, tree): + def __serialize_xml(self, tree): """ Serialize a XML document. Basic wrapper around lxml.tostring() """ return ET.tostring(tree, with_tail=False, xml_declaration=True, encoding=tree.docinfo.encoding) - def __ver (self, version): + def __ver(self, version): """ Helper for displaying lxml version numbers """ return ".".join(map(str, version)) - def reset (self): - + def reset(self): + """ Reset the mutator """ self.tree = deepcopy(self.input_tree) - def init_from_string (self, input_string): - + def init_from_string(self, input_string): + """ Initialize the mutator from a XML string """ # Get a pointer to the top-element @@ -97,15 +98,15 @@ class XmlMutatorMin: # Get a working copy self.tree = deepcopy(self.input_tree) - def save_to_string (self): - + def save_to_string(self): + """ Return the current XML document as UTF-8 string """ # Return a text version of the tree return self.__serialize_xml(self.tree) - def __pick_element (self, exclude_root_node = False): - + def __pick_element(self, exclude_root_node=False): + """ Pick a random element from the current document """ # Get a list of all elements, but nodes like PI and comments @@ -119,7 +120,7 @@ class XmlMutatorMin: # Pick a random element try: - elem_id = random.randint (start, len(elems) - 1) + elem_id = random.randint(start, len(elems) - 1) elem = elems[elem_id] except ValueError: # Should only occurs if "exclude_root_node = True" @@ -127,8 +128,8 @@ class XmlMutatorMin: return (elem_id, elem) - def __fuzz_attribute (self): - + def __fuzz_attribute(self): + """ Fuzz (part of) an attribute value """ # Select a node to modify @@ -144,19 +145,19 @@ class XmlMutatorMin: return # Pick a random attribute - rand_attrib_id = random.randint (0, len(attribs) - 1) + rand_attrib_id = random.randint(0, len(attribs) - 1) rand_attrib = attribs[rand_attrib_id] # We have the attribute to modify # Get its value - attrib_value = rand_elem.get(rand_attrib); + attrib_value = rand_elem.get(rand_attrib) # print("- Value: " + attrib_value) # Should we work on the whole value? func_call = "(?P<func>[a-zA-Z:\-]+)\((?P<args>.*?)\)" p = re.compile(func_call) l = p.findall(attrib_value) - if random.choice((True,False)) and l: + if random.choice((True, False)) and l: # Randomly pick one the function calls (func, args) = random.choice(l) # Split by "," and randomly pick one of the arguments @@ -236,29 +237,29 @@ class XmlMutatorMin: # Modify the attribute rand_elem.set(rand_attrib, new_value.decode("utf-8")) - def __del_node_and_children (self): + def __del_node_and_children(self): """ High-level minimizing mutator Delete a random node and its children (i.e. delete a random tree) """ self.__del_node(True) - def __del_node_but_children (self): + def __del_node_but_children(self): """ High-level minimizing mutator Delete a random node but its children (i.e. link them to the parent of the deleted node) """ self.__del_node(False) - def __del_node (self, delete_children): - + def __del_node(self, delete_children): + """ Called by the __del_node_* mutators """ # Select a node to modify (but the root one) - (rand_elem_id, rand_elem) = self.__pick_element (exclude_root_node = True) + (rand_elem_id, rand_elem) = self.__pick_element(exclude_root_node=True) # If the document includes only a top-level element - # Then we can't pick a element (given that "exclude_root_node = True") + # Then we can't pick a element (given that "exclude_root_node = True") # Is the document deep enough? if rand_elem is None: @@ -275,12 +276,12 @@ class XmlMutatorMin: # Link children of the random (soon to be deleted) node to its parent for child in rand_elem: rand_elem.getparent().append(child) - + # Remove the node rand_elem.getparent().remove(rand_elem) - def __del_content (self): - + def __del_content(self): + """ High-level minimizing mutator Delete the attributes and children of a random node """ @@ -294,8 +295,8 @@ class XmlMutatorMin: # Reset the node rand_elem.clear() - def __del_attribute (self): - + def __del_attribute(self): + """ High-level minimizing mutator Delete a random attribute from a random node """ @@ -312,7 +313,7 @@ class XmlMutatorMin: return # Pick a random attribute - rand_attrib_id = random.randint (0, len(attribs) - 1) + rand_attrib_id = random.randint(0, len(attribs) - 1) rand_attrib = attribs[rand_attrib_id] # Log something @@ -322,8 +323,8 @@ class XmlMutatorMin: # Delete the attribute rand_elem.attrib.pop(rand_attrib) - def mutate (self, min=1, max=5): - + def mutate(self, min=1, max=5): + """ Execute some high-level mutators between $min and $max times, then some medium-level ones """ # High-level mutation diff --git a/examples/python_mutators/common.py b/examples/custom_mutators/common.py index 28b8ee80..9a1ef0a3 100644 --- a/examples/python_mutators/common.py +++ b/examples/custom_mutators/common.py @@ -19,19 +19,22 @@ import random import os import re + def randel(l): if not l: return None - return l[random.randint(0,len(l)-1)] + return l[random.randint(0, len(l)-1)] + def randel_pop(l): if not l: return None - return l.pop(random.randint(0,len(l)-1)) + return l.pop(random.randint(0, len(l)-1)) + def write_exc_example(data, exc): exc_name = re.sub(r'[^a-zA-Z0-9]', '_', repr(exc)) - + if not os.path.exists(exc_name): with open(exc_name, 'w') as f: - f.write(data) + f.write(data) diff --git a/examples/custom_mutators/example.c b/examples/custom_mutators/example.c new file mode 100644 index 00000000..63e4d6da --- /dev/null +++ b/examples/custom_mutators/example.c @@ -0,0 +1,177 @@ +/* + New Custom Mutator for AFL++ + Written by Khaled Yakdan <yakdan@code-intelligence.de> + Andrea Fioraldi <andreafioraldi@gmail.com> + Shengtuo Hu <h1994st@gmail.com> +*/ + +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +static const char *commands[] = { + + "GET", + "PUT", + "DEL", + +}; + +static size_t data_size = 100; + +void afl_custom_init(unsigned int seed) { + + srand(seed); + +} + +/** + * Perform custom mutations on a given input + * + * (Optional for now. Required in the future) + * + * @param[in] buf Input data to be mutated + * @param[in] buf_size Size of input data + * @param[in] add_buf Buffer containing the additional test case + * @param[in] add_buf_size Size of the additional test case + * @param[out] mutated_out Buffer to store the mutated input + * @param[in] max_size Maximum size of the mutated output. The mutation must not + * produce data larger than max_size. + * @return Size of the mutated output. + */ +size_t afl_custom_fuzz(uint8_t *buf, size_t buf_size, + uint8_t *add_buf,size_t add_buf_size, // add_buf can be NULL + uint8_t *mutated_out, size_t max_size) { + + // Make sure that the packet size does not exceed the maximum size expected by + // the fuzzer + size_t mutated_size = data_size <= max_size ? data_size : max_size; + + // Randomly select a command string to add as a header to the packet + memcpy(mutated_out, commands[rand() % 3], 3); + + // Mutate the payload of the packet + for (int i = 3; i < mutated_size; i++) { + + mutated_out[i] = (data[i] + rand() % 10) & 0xff; + + } + + return mutated_size; + +} + +/** + * A post-processing function to use right before AFL writes the test case to + * disk in order to execute the target. + * + * (Optional) If this functionality is not needed, simply don't define this + * function. + * + * @param[in] buf Buffer containing the test case to be executed + * @param[in] buf_size Size of the test case + * @param[out] out_buf Pointer to the buffer containing the test case after + * processing. External library should allocate memory for out_buf. AFL++ + * will release the memory after saving the test case. + * @return Size of the output buffer after processing + */ +size_t afl_custom_pre_save(uint8_t *buf, size_t buf_size, uint8_t **out_buf) { + + size_t out_buf_size; + + out_buf_size = buf_size; + + // External mutator should allocate memory for `out_buf` + *out_buf = malloc(out_buf_size); + memcpy(*out_buf, buf, out_buf_size); + + return out_buf_size; + +} + +uint8_t *trim_buf; +size_t trim_buf_size +int trimmming_steps; +int cur_step; + +/** + * This method is called at the start of each trimming operation and receives + * the initial buffer. It should return the amount of iteration steps possible + * on this input (e.g. if your input has n elements and you want to remove + * them one by one, return n, if you do a binary search, return log(n), + * and so on...). + * + * If your trimming algorithm doesn't allow you to determine the amount of + * (remaining) steps easily (esp. while running), then you can alternatively + * return 1 here and always return 0 in post_trim until you are finished and + * no steps remain. In that case, returning 1 in post_trim will end the + * trimming routine. The whole current index/max iterations stuff is only used + * to show progress. + * + * (Optional) + * + * @param buf Buffer containing the test case + * @param buf_size Size of the test case + * @return The amount of possible iteration steps to trim the input + */ +int afl_custom_init_trim(uint8_t *buf, size_t buf_size) { + + // We simply trim once + trimmming_steps = 1; + + cur_step = 0; + trim_buf = buf; + trim_buf_size = buf_size; + + return trimmming_steps; + +} + +/** + * This method is called for each trimming operation. It doesn't have any + * arguments because we already have the initial buffer from init_trim and we + * can memorize the current state in global variables. This can also save + * reparsing steps for each iteration. It should return the trimmed input + * buffer, where the returned data must not exceed the initial input data in + * length. Returning anything that is larger than the original data (passed + * to init_trim) will result in a fatal abort of AFLFuzz. + * + * (Optional) + * + * @param[out] out_buf Pointer to the buffer containing the trimmed test case. + * External library should allocate memory for out_buf. AFL++ will release + * the memory after saving the test case. + * @param[out] out_buf_size Pointer to the size of the trimmed test case + */ +void afl_custom_trim(uint8_t **out_buf, size_t* out_buf_size) { + + *out_buf_size = trim_buf_size - 1; + + // External mutator should allocate memory for `out_buf` + *out_buf = malloc(*out_buf_size); + // Remove the last byte of the trimming input + memcpy(*out_buf, trim_buf, *out_buf_size); + +} + +/** + * This method is called after each trim operation to inform you if your + * trimming step was successful or not (in terms of coverage). If you receive + * a failure here, you should reset your input to the last known good state. + * + * (Optional) + * + * @param success Indicates if the last trim operation was successful. + * @return The next trim iteration index (from 0 to the maximum amount of + * steps returned in init_trim) + */ +int afl_custom_post_trim(int success) { + + if (success) { + ++cur_step; + return cur_step; + } + + return trimmming_steps; + +} diff --git a/examples/python_mutators/example.py b/examples/custom_mutators/example.py index d32a7eb2..a68f2ee5 100644 --- a/examples/python_mutators/example.py +++ b/examples/custom_mutators/example.py @@ -16,26 +16,31 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/. import random + def init(seed): ''' Called once when AFLFuzz starts up. Used to seed our RNG. - + @type seed: int @param seed: A 32-bit random value ''' random.seed(seed) - return 0 -def fuzz(buf, add_buf): + +def fuzz(buf, add_buf, max_size): ''' Called per fuzzing iteration. - + @type buf: bytearray @param buf: The buffer that should be mutated. - + @type add_buf: bytearray @param add_buf: A second buffer that can be used as mutation source. - + + @type max_size: int + @param max_size: Maximum size of the mutated output. The mutation must not + produce data larger than max_size. + @rtype: bytearray @return: A new bytearray containing the mutated data ''' @@ -50,54 +55,68 @@ def fuzz(buf, add_buf): # def init_trim(buf): # ''' # Called per trimming iteration. -# +# # @type buf: bytearray # @param buf: The buffer that should be trimmed. -# +# # @rtype: int # @return: The maximum number of trimming steps. # ''' # global ... -# +# # # Initialize global variables -# +# # # Figure out how many trimming steps are possible. # # If this is not possible for your trimming, you can # # return 1 instead and always return 0 in post_trim # # until you are done (then you return 1). -# +# # return steps -# +# # def trim(): # ''' # Called per trimming iteration. -# +# # @rtype: bytearray # @return: A new bytearray containing the trimmed data. # ''' # global ... -# +# # # Implement the actual trimming here -# +# # return bytearray(...) -# +# # def post_trim(success): # ''' # Called after each trimming operation. -# +# # @type success: bool # @param success: Indicates if the last trim operation was successful. -# +# # @rtype: int # @return: The next trim index (0 to max number of steps) where max # number of steps indicates the trimming is done. # ''' # global ... -# +# # if not success: # # Restore last known successful input, determine next index # else: # # Just determine the next index, based on what was successfully # # removed in the last step -# +# # return next_index +# +# def pre_save(buf): +# ''' +# Called just before the execution to write the test case in the format +# expected by the target +# +# @type buf: bytearray +# @param buf: The buffer containing the test case to be executed +# +# @rtype: bytearray +# @return: The buffer containing the test case after +# ''' +# return buf +# diff --git a/examples/python_mutators/simple-chunk-replace.py b/examples/custom_mutators/simple-chunk-replace.py index 218dd4f8..df2f4ca7 100644 --- a/examples/python_mutators/simple-chunk-replace.py +++ b/examples/custom_mutators/simple-chunk-replace.py @@ -16,27 +16,32 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/. import random + def init(seed): ''' Called once when AFLFuzz starts up. Used to seed our RNG. - + @type seed: int @param seed: A 32-bit random value ''' # Seed our RNG random.seed(seed) - return 0 -def fuzz(buf, add_buf): + +def fuzz(buf, add_buf, max_size): ''' Called per fuzzing iteration. - + @type buf: bytearray @param buf: The buffer that should be mutated. - + @type add_buf: bytearray @param add_buf: A second buffer that can be used as mutation source. - + + @type max_size: int + @param max_size: Maximum size of the mutated output. The mutation must not + produce data larger than max_size. + @rtype: bytearray @return: A new bytearray containing the mutated data ''' @@ -45,10 +50,10 @@ def fuzz(buf, add_buf): # Take a random fragment length between 2 and 32 (or less if add_buf is shorter) fragment_len = random.randint(1, min(len(add_buf), 32)) - + # Determine a random source index where to take the data chunk from rand_src_idx = random.randint(0, len(add_buf) - fragment_len) - + # Determine a random destination index where to put the data chunk rand_dst_idx = random.randint(0, len(buf)) diff --git a/examples/custom_mutators/simple_mutator.c b/examples/custom_mutators/simple_mutator.c deleted file mode 100644 index bf655679..00000000 --- a/examples/custom_mutators/simple_mutator.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - Simple Custom Mutator for AFL - - Written by Khaled Yakdan <yakdan@code-intelligence.de> - - This a simple mutator that assumes that the generates messages starting with - one of the three strings GET, PUT, or DEL followed by a payload. The mutator - randomly selects a commend and mutates the payload of the seed provided as - input. -*/ - -#include <stdint.h> -#include <stdlib.h> -#include <string.h> - -static const char *commands[] = { - - "GET", - "PUT", - "DEL", - -}; - -static size_t data_size = 100; - -size_t afl_custom_mutator(uint8_t *data, size_t size, uint8_t *mutated_out, - size_t max_size, unsigned int seed) { - - // Seed the PRNG - srand(seed); - - // Make sure that the packet size does not exceed the maximum size expected by - // the fuzzer - size_t mutated_size = data_size <= max_size ? data_size : max_size; - - // Randomly select a command string to add as a header to the packet - memcpy(mutated_out, commands[rand() % 3], 3); - - // Mutate the payload of the packet - for (int i = 3; i < mutated_size; i++) { - - mutated_out[i] = (data[i] + rand() % 10) & 0xff; - - } - - return mutated_size; - -} - diff --git a/examples/python_mutators/wrapper_afl_min.py b/examples/custom_mutators/wrapper_afl_min.py index df09b40a..ecb03b55 100644 --- a/examples/python_mutators/wrapper_afl_min.py +++ b/examples/custom_mutators/wrapper_afl_min.py @@ -9,11 +9,11 @@ __seed__ = "RANDOM" __log__ = False __log_file__ = "wrapper.log" -# AFL functions +# AFL functions def log(text): """ - Logger + Logger """ global __seed__ @@ -24,6 +24,7 @@ def log(text): with open(__log_file__, "a") as logf: logf.write("[%s] %s\n" % (__seed__, text)) + def init(seed): """ Called once when AFL starts up. Seed is used to identify the AFL instance in log files @@ -37,17 +38,18 @@ def init(seed): # Create a global mutation class try: - __mutator__ = XmlMutatorMin(__seed__, verbose=__log__) + __mutator__ = XmlMutatorMin(__seed__, verbose=__log__) log("init(): Mutator created") except RuntimeError as e: log("init(): Can't create mutator: %s" % e.message) -def fuzz(buf, add_buf): + +def fuzz(buf, add_buf, max_size): """ - Called for each fuzzing iteration. + Called for each fuzzing iteration. """ - global __mutator__ + global __mutator__ # Do we have a working mutator object? if __mutator__ is None: @@ -62,7 +64,7 @@ def fuzz(buf, add_buf): try: buf_str = str(buf) log("fuzz(): AFL buffer converted to a string") - except: + except Exception: via_buffer = False log("fuzz(): Can't convert AFL buffer to a string") @@ -71,28 +73,28 @@ def fuzz(buf, add_buf): try: __mutator__.init_from_string(buf_str) log("fuzz(): Mutator successfully initialized with AFL buffer (%d bytes)" % len(buf_str)) - except: + except Exception: via_buffer = False log("fuzz(): Can't initialize mutator with AFL buffer") # If init from AFL buffer wasn't succesful if not via_buffer: - log("fuzz(): Returning unmodified AFL buffer") - return buf + log("fuzz(): Returning unmodified AFL buffer") + return buf # Sucessful initialization -> mutate try: __mutator__.mutate(max=5) log("fuzz(): Input mutated") - except: + except Exception: log("fuzz(): Can't mutate input => returning buf") return buf - + # Convert mutated data to a array of bytes try: data = bytearray(__mutator__.save_to_string()) log("fuzz(): Mutated data converted as bytes") - except: + except Exception: log("fuzz(): Can't convert mutated data to bytes => returning buf") return buf @@ -100,8 +102,8 @@ def fuzz(buf, add_buf): log("fuzz(): Returning %d bytes" % len(data)) return data -# Main (for debug) +# Main (for debug) if __name__ == '__main__': __log__ = True @@ -114,4 +116,3 @@ if __name__ == '__main__': in_2 = bytearray("<abc abc123='456' abcCBA='ppppppppppppppppppppppppppppp'/>") out = fuzz(in_1, in_2) print(out) - diff --git a/examples/python_mutators/README b/examples/python_mutators/README deleted file mode 100644 index 8e378405..00000000 --- a/examples/python_mutators/README +++ /dev/null @@ -1,18 +0,0 @@ -These are example and helper files for the AFL_PYTHON_MODULE feature. -See [docs/python_mutators.md](../docs/python_mutators.md) for more information - -Note that if you compile with python3.7 you must use python3 scripts, and if -you use pyton2.7 to compile python2 scripts! - - -example.py - this is the template you can use, the functions are there - but they are empty - -simple-chunk-replace.py - this is a simple example where chunks are replaced - -common.py - this can be used for common functions and helpers. - the examples do not use this though. But you can :) - -wrapper_afl_min.py - mutation of XML documents, loads XmlMutatorMin.py - -XmlMutatorMin.py - module for XML mutation diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 1999f16c..37b6832c 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -276,8 +276,7 @@ extern u8 cal_cycles, /* Calibration cycles defaults */ no_unlink, /* do not unlink cur_input */ use_stdin, /* use stdin for sending data */ debug, /* Debug mode */ - custom_only, /* Custom mutator only mode */ - python_only; /* Python-only mode */ + custom_only; /* Custom mutator only mode */ extern u32 stats_update_freq; /* Stats update frequency (execs) */ @@ -459,30 +458,112 @@ u8* (*post_handler)(u8* buf, u32* len); extern u8* cmplog_binary; extern s32 cmplog_child_pid, cmplog_forksrv_pid; -/* hooks for the custom mutator function */ -/** - * Perform custom mutations on a given input - * @param data Input data to be mutated - * @param size Size of input data - * @param mutated_out Buffer to store the mutated input - * @param max_size Maximum size of the mutated output. The mutation must not - * produce data larger than max_size. - * @param seed Seed used for the mutation. The mutation should produce the same - * output given the same seed. - * @return Size of the mutated output. - */ -size_t (*custom_mutator)(u8* data, size_t size, u8* mutated_out, - size_t max_size, unsigned int seed); -/** - * A post-processing function to use right before AFL writes the test case to - * disk in order to execute the target. If this functionality is not needed, - * Simply don't define this function. - * @param data Buffer containing the test case to be executed. - * @param size Size of the test case. - * @param new_data Buffer to store the test case after processing - * @return Size of data after processing. - */ -size_t (*pre_save_handler)(u8* data, size_t size, u8** new_data); +/* Custom mutators */ + +struct custom_mutator { + const char* name; + void* dh; + + /* hooks for the custom mutator function */ + + /** + * Initialize the custom mutator. + * + * (Optional) + * + * @param seed Seed used for the mutation. + */ + void (*afl_custom_init)(unsigned int seed); + + /** + * Perform custom mutations on a given input + * + * (Optional for now. Required in the future) + * + * @param[in] buf Input data to be mutated + * @param[in] buf_size Size of input data + * @param[in] add_buf Buffer containing the additional test case + * @param[in] add_buf_size Size of the additional test case + * @param[out] mutated_out Buffer to store the mutated input + * @param[in] max_size Maximum size of the mutated output. The mutation must not + * produce data larger than max_size. + * @return Size of the mutated output. + */ + size_t (*afl_custom_fuzz)(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size); + + /** + * A post-processing function to use right before AFL writes the test case to + * disk in order to execute the target. + * + * (Optional) If this functionality is not needed, simply don't define this + * function. + * + * @param[in] buf Buffer containing the test case to be executed + * @param[in] buf_size Size of the test case + * @param[out] out_buf Pointer to the buffer of storing the test case after + * processing. External library should allocate memory for out_buf. AFL++ + * will release the memory after saving the test case. + * @return Size of the output buffer after processing + */ + size_t (*afl_custom_pre_save)(u8* buf, size_t buf_size, u8** out_buf); + + /** + * This method is called at the start of each trimming operation and receives + * the initial buffer. It should return the amount of iteration steps possible + * on this input (e.g. if your input has n elements and you want to remove + * them one by one, return n, if you do a binary search, return log(n), + * and so on...). + * + * If your trimming algorithm doesn't allow you to determine the amount of + * (remaining) steps easily (esp. while running), then you can alternatively + * return 1 here and always return 0 in post_trim until you are finished and + * no steps remain. In that case, returning 1 in post_trim will end the + * trimming routine. The whole current index/max iterations stuff is only used + * to show progress. + * + * (Optional) + * + * @param buf Buffer containing the test case + * @param buf_size Size of the test case + * @return The amount of possible iteration steps to trim the input + */ + u32 (*afl_custom_init_trim)(u8* buf, size_t buf_size); + + /** + * This method is called for each trimming operation. It doesn't have any + * arguments because we already have the initial buffer from init_trim and we + * can memorize the current state in global variables. This can also save + * reparsing steps for each iteration. It should return the trimmed input + * buffer, where the returned data must not exceed the initial input data in + * length. Returning anything that is larger than the original data (passed + * to init_trim) will result in a fatal abort of AFLFuzz. + * + * (Optional) + * + * @param[out] out_buf Pointer to the buffer containing the trimmed test case. + * External library should allocate memory for out_buf. AFL++ will release + * the memory after saving the test case. + * @param[out] out_buf_size Pointer to the size of the trimmed test case + */ + void (*afl_custom_trim)(u8** out_buf, size_t* out_buf_size); + + /** + * This method is called after each trim operation to inform you if your + * trimming step was successful or not (in terms of coverage). If you receive + * a failure here, you should reset your input to the last known good state. + * + * (Optional) + * + * @param success Indicates if the last trim operation was successful. + * @return The next trim iteration index (from 0 to the maximum amount of + * steps returned in init_trim) + */ + u32 (*afl_custom_post_trim)(u8 success); +}; + +extern struct custom_mutator* mutator; /* Interesting values, as per config.h */ @@ -525,9 +606,10 @@ enum { /* 00 */ PY_FUNC_INIT, /* 01 */ PY_FUNC_FUZZ, - /* 02 */ PY_FUNC_INIT_TRIM, - /* 03 */ PY_FUNC_POST_TRIM, - /* 04 */ PY_FUNC_TRIM, + /* 02 */ PY_FUNC_PRE_SAVE, + /* 03 */ PY_FUNC_INIT_TRIM, + /* 04 */ PY_FUNC_POST_TRIM, + /* 05 */ PY_FUNC_TRIM, PY_FUNC_COUNT }; @@ -538,15 +620,26 @@ extern PyObject* py_functions[PY_FUNC_COUNT]; /**** Prototypes ****/ +/* Custom mutators */ +void setup_custom_mutator(void); +void destroy_custom_mutator(void); +void load_custom_mutator(const char*); +void load_custom_mutator_py(const char*); +u8 trim_case_custom(char** argv, struct queue_entry* q, u8* in_buf); + /* Python */ #ifdef USE_PYTHON -int init_py(); -void finalize_py(); -void fuzz_py(char*, size_t, char*, size_t, char**, size_t*); -u32 init_trim_py(char*, size_t); -u32 post_trim_py(char); -void trim_py(char**, size_t*); -u8 trim_case_python(char**, struct queue_entry*, u8*); +int init_py_module(u8*); +void finalize_py_module(); + +void init_py(unsigned int seed); +size_t fuzz_py(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size); +size_t pre_save_py(u8* data, size_t size, u8** new_data); +u32 init_trim_py(u8*, size_t); +u32 post_trim_py(u8); +void trim_py(u8**, size_t*); #endif /* Queue */ @@ -629,7 +722,6 @@ u8 fuzz_one(char**); void bind_to_free_cpu(void); #endif void setup_post(void); -void setup_custom_mutator(void); void read_testcases(void); void perform_dry_run(char**); void pivot_inputs(void); diff --git a/include/envs.h b/include/envs.h index 306143be..791887d7 100644 --- a/include/envs.h +++ b/include/envs.h @@ -24,7 +24,7 @@ const char *afl_environment_variables[] = { "AFL_NO_X86", // not really an env but we dont want to warn on it "AFL_PATH", "AFL_PERFORMANCE_FILE", //"AFL_PERSISTENT", // not implemented anymore, so warn additionally - "AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE", "AFL_PYTHON_ONLY", + "AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE", "AFL_QEMU_COMPCOV", "AFL_QEMU_COMPCOV_DEBUG", "AFL_QEMU_DEBUG_MAPS", "AFL_QEMU_DISABLE_CACHE", "AFL_QEMU_PERSISTENT_ADDR", "AFL_QEMU_PERSISTENT_CNT", "AFL_QEMU_PERSISTENT_GPR", diff --git a/src/afl-fuzz-globals.c b/src/afl-fuzz-globals.c index ae343026..87418753 100644 --- a/src/afl-fuzz-globals.c +++ b/src/afl-fuzz-globals.c @@ -88,8 +88,7 @@ u8 cal_cycles = CAL_CYCLES, /* Calibration cycles defaults */ no_unlink, /* do not unlink cur_input */ use_stdin = 1, /* use stdin for sending data */ be_quiet, /* is AFL_QUIET set? */ - custom_only, /* Custom mutator only mode */ - python_only; /* Python-only mode */ + custom_only; /* Custom mutator only mode */ u32 stats_update_freq = 1; /* Stats update frequency (execs) */ @@ -256,10 +255,8 @@ u8 *(*post_handler)(u8 *buf, u32 *len); u8 *cmplog_binary; s32 cmplog_child_pid, cmplog_forksrv_pid; -/* hooks for the custom mutator function */ -size_t (*custom_mutator)(u8 *data, size_t size, u8 *mutated_out, - size_t max_size, unsigned int seed); -size_t (*pre_save_handler)(u8 *data, size_t size, u8 **new_data); +/* Custom mutator */ +struct custom_mutator* mutator; /* Interesting values, as per config.h */ diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index a82fa8f9..08b6de60 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -296,34 +296,6 @@ void setup_post(void) { } -void setup_custom_mutator(void) { - - void* dh; - u8* fn = getenv("AFL_CUSTOM_MUTATOR_LIBRARY"); - - if (!fn) return; - - if (limit_time_sig) - FATAL( - "MOpt and custom mutator are mutually exclusive. We accept pull " - "requests that integrates MOpt with the optional mutators " - "(custom/radamsa/redquenn/...)."); - - ACTF("Loading custom mutator library from '%s'...", fn); - - dh = dlopen(fn, RTLD_NOW); - if (!dh) FATAL("%s", dlerror()); - - custom_mutator = dlsym(dh, "afl_custom_mutator"); - if (!custom_mutator) FATAL("Symbol 'afl_custom_mutator' not found."); - - pre_save_handler = dlsym(dh, "afl_pre_save_handler"); - // if (!pre_save_handler) WARNF("Symbol 'afl_pre_save_handler' not found."); - - OKF("Custom mutator installed successfully."); - -} - /* Shuffle an array of pointers. Might be slightly biased. */ static void shuffle_ptrs(void** ptrs, u32 cnt) { diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c new file mode 100644 index 00000000..26eaea59 --- /dev/null +++ b/src/afl-fuzz-mutators.c @@ -0,0 +1,311 @@ +/* + american fuzzy lop++ - custom mutators related routines + ------------------------------------------------------- + + Originally written by Shengtuo Hu + + Now maintained by Marc Heuse <mh@mh-sec.de>, + Heiko Eißfeldt <heiko.eissfeldt@hexco.de> and + Andrea Fioraldi <andreafioraldi@gmail.com> + + Copyright 2016, 2017 Google Inc. All rights reserved. + Copyright 2019-2020 AFLplusplus Project. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + + This is the real deal: the program takes an instrumented binary and + attempts a variety of basic fuzzing tricks, paying close attention to + how they affect the execution path. + + */ + +#include "afl-fuzz.h" + +void setup_custom_mutator(void) { + + /* Try mutator library first */ + u8* fn = getenv("AFL_CUSTOM_MUTATOR_LIBRARY"); + + if (fn) { + if (limit_time_sig) + FATAL( + "MOpt and custom mutator are mutually exclusive. We accept pull " + "requests that integrates MOpt with the optional mutators " + "(custom/radamsa/redquenn/...)."); + + load_custom_mutator(fn); + + return; + } + + /* Try Python module */ +#ifdef USE_PYTHON + u8* module_name = getenv("AFL_PYTHON_MODULE"); + + if (module_name) { + + if (limit_time_sig) + FATAL( + "MOpt and Python mutator are mutually exclusive. We accept pull " + "requests that integrates MOpt with the optional mutators " + "(custom/radamsa/redquenn/...)."); + + if (init_py_module(module_name)) + FATAL("Failed to initialize Python module"); + + load_custom_mutator_py(module_name); + + } +#else + if (getenv("AFL_PYTHON_MODULE")) + FATAL("Your AFL binary was built without Python support"); +#endif + +} + +void destroy_custom_mutator(void) { + + if (mutator) { + if (mutator->dh) + dlclose(mutator->dh); + else { + /* Python mutator */ +#ifdef USE_PYTHON + finalize_py_module(); +#endif + } + + ck_free(mutator); + } + +} + +void load_custom_mutator(const char* fn) { + + void* dh; + mutator = ck_alloc(sizeof(struct custom_mutator)); + + mutator->name = fn; + ACTF("Loading custom mutator library from '%s'...", fn); + + dh = dlopen(fn, RTLD_NOW); + if (!dh) FATAL("%s", dlerror()); + mutator->dh = dh; + + /* Mutator */ + /* "afl_custom_init", optional for backward compatibility */ + mutator->afl_custom_init = dlsym(dh, "afl_custom_init"); + if (!mutator->afl_custom_init) + WARNF("Symbol 'afl_custom_init' not found."); + + /* "afl_custom_fuzz" or "afl_custom_mutator", required */ + mutator->afl_custom_fuzz = dlsym(dh, "afl_custom_fuzz"); + if (!mutator->afl_custom_fuzz) { + + /* Try "afl_custom_mutator" for backward compatibility */ + WARNF("Symbol 'afl_custom_fuzz' not found. Try 'afl_custom_mutator'."); + + mutator->afl_custom_fuzz = dlsym(dh, "afl_custom_mutator"); + if (!mutator->afl_custom_fuzz) + FATAL("Symbol 'afl_custom_mutator' not found."); + + } + + /* "afl_custom_pre_save", optional */ + mutator->afl_custom_pre_save = dlsym(dh, "afl_custom_pre_save"); + if (!mutator->afl_custom_pre_save) + WARNF("Symbol 'afl_custom_pre_save' not found."); + + u8 notrim = 0; + /* "afl_custom_init_trim", optional */ + mutator->afl_custom_init_trim = dlsym(dh, "afl_custom_init_trim"); + if (!mutator->afl_custom_init_trim) + WARNF("Symbol 'afl_custom_init_trim' not found."); + + /* "afl_custom_trim", optional */ + mutator->afl_custom_trim = dlsym(dh, "afl_custom_trim"); + if (!mutator->afl_custom_trim) + WARNF("Symbol 'afl_custom_trim' not found."); + + /* "afl_custom_post_trim", optional */ + mutator->afl_custom_post_trim = dlsym(dh, "afl_custom_post_trim"); + if (!mutator->afl_custom_post_trim) + WARNF("Symbol 'afl_custom_post_trim' not found."); + + if (notrim) { + + mutator->afl_custom_init_trim = NULL; + mutator->afl_custom_trim = NULL; + mutator->afl_custom_post_trim = NULL; + WARNF( + "Custom mutator does not implement all three trim APIs, standard " + "trimming will be used."); + + } + + OKF("Custom mutator '%s' installed successfully.", fn); + + /* Initialize the custom mutator */ + if (mutator->afl_custom_init) + mutator->afl_custom_init(UR(0xFFFFFFFF)); + +} + +u8 trim_case_custom(char** argv, struct queue_entry* q, u8* in_buf) { + + static u8 tmp[64]; + static u8 clean_trace[MAP_SIZE]; + + u8 needs_write = 0, fault = 0; + u32 trim_exec = 0; + u32 orig_len = q->len; + + stage_name = tmp; + bytes_trim_in += q->len; + + /* Initialize trimming in the custom mutator */ + stage_cur = 0; + stage_max = mutator->afl_custom_init_trim(in_buf, q->len); + + if (not_on_tty && debug) + SAYF("[Custom Trimming] START: Max %d iterations, %u bytes", stage_max, + q->len); + + while (stage_cur < stage_max) { + + sprintf(tmp, "ptrim %s", DI(trim_exec)); + + u32 cksum; + + u8* retbuf = NULL; + size_t retlen = 0; + + mutator->afl_custom_trim(&retbuf, &retlen); + + if (retlen > orig_len) + FATAL( + "Trimmed data returned by custom mutator is larger than original " + "data"); + + write_to_testcase(retbuf, retlen); + + fault = run_target(argv, exec_tmout); + ++trim_execs; + + if (stop_soon || fault == FAULT_ERROR) { + + free(retbuf); + goto abort_trimming; + + } + + cksum = hash32(trace_bits, MAP_SIZE, HASH_CONST); + + if (cksum == q->exec_cksum) { + + q->len = retlen; + memcpy(in_buf, retbuf, retlen); + + /* Let's save a clean trace, which will be needed by + update_bitmap_score once we're done with the trimming stuff. */ + + if (!needs_write) { + + needs_write = 1; + memcpy(clean_trace, trace_bits, MAP_SIZE); + + } + + /* Tell the custom mutator that the trimming was successful */ + stage_cur = mutator->afl_custom_post_trim(1); + + if (not_on_tty && debug) + SAYF("[Custom Trimming] SUCCESS: %d/%d iterations (now at %u bytes)", + stage_cur, stage_max, q->len); + + } else { + + /* Tell the custom mutator that the trimming was unsuccessful */ + stage_cur = mutator->afl_custom_post_trim(0); + if (not_on_tty && debug) + SAYF("[Custom Trimming] FAILURE: %d/%d iterations", stage_cur, + stage_max); + + } + + free(retbuf); + + /* Since this can be slow, update the screen every now and then. */ + + if (!(trim_exec++ % stats_update_freq)) show_stats(); + + } + + if (not_on_tty && debug) + SAYF("[Custom Trimming] DONE: %u bytes -> %u bytes", orig_len, q->len); + + /* If we have made changes to in_buf, we also need to update the on-disk + version of the test case. */ + + if (needs_write) { + + s32 fd; + + unlink(q->fname); /* ignore errors */ + + fd = open(q->fname, O_WRONLY | O_CREAT | O_EXCL, 0600); + + if (fd < 0) PFATAL("Unable to create '%s'", q->fname); + + ck_write(fd, in_buf, q->len, q->fname); + close(fd); + + memcpy(trace_bits, clean_trace, MAP_SIZE); + update_bitmap_score(q); + + } + +abort_trimming: + + bytes_trim_out += q->len; + return fault; + +} + +void load_custom_mutator_py(const char* module_name) { + + mutator = ck_alloc(sizeof(struct custom_mutator)); + + mutator->name = module_name; + ACTF("Loading Python mutator library from '%s'...", module_name); + + if (py_functions[PY_FUNC_INIT]) + mutator->afl_custom_init = init_py; + + /* "afl_custom_fuzz" should not be NULL, but the interface of Python mutator + is quite different from the custom mutator. */ + mutator->afl_custom_fuzz = fuzz_py; + + if (py_functions[PY_FUNC_PRE_SAVE]) + mutator->afl_custom_pre_save = pre_save_py; + + if (py_functions[PY_FUNC_INIT_TRIM]) + mutator->afl_custom_init_trim = init_trim_py; + + if (py_functions[PY_FUNC_POST_TRIM]) + mutator->afl_custom_post_trim = post_trim_py; + + if (py_functions[PY_FUNC_TRIM]) + mutator->afl_custom_trim = trim_py; + + OKF("Python mutator '%s' installed successfully.", module_name); + + /* Initialize the custom mutator */ + if (mutator->afl_custom_init) + mutator->afl_custom_init(UR(0xFFFFFFFF)); + +} diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index f1efe2df..5d00e8df 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -449,7 +449,7 @@ u8 fuzz_one_original(char** argv) { * TRIMMING * ************/ - if (!dumb_mode && !queue_cur->trim_done && !custom_mutator && !disable_trim) { + if (!dumb_mode && !queue_cur->trim_done && !disable_trim) { u8 res = trim_case(argv, queue_cur, in_buf); @@ -482,55 +482,6 @@ u8 fuzz_one_original(char** argv) { if (use_radamsa > 1) goto radamsa_stage; - // custom_stage: // not used - yet - - if (custom_mutator) { - - stage_short = "custom"; - stage_name = "custom mutator"; - stage_max = len << 3; - stage_val_type = STAGE_VAL_NONE; - - const u32 max_seed_size = 4096 * 4096; - u8* mutated_buf = ck_alloc(max_seed_size); - - orig_hit_cnt = queued_paths + unique_crashes; - - for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) { - - size_t orig_size = (size_t)len; - size_t mutated_size = custom_mutator(in_buf, orig_size, mutated_buf, - max_seed_size, UR(UINT32_MAX)); - if (mutated_size > 0) { - - out_buf = ck_realloc(out_buf, mutated_size); - memcpy(out_buf, mutated_buf, mutated_size); - if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) { - - goto abandon_entry; - - } - - } - - } - - ck_free(mutated_buf); - new_hit_cnt = queued_paths + unique_crashes; - - stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt; - stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max; - - if (custom_only) { - - /* Skip other stages */ - ret_val = 0; - goto abandon_entry; - - } - - } - if (cmplog_mode) { if (input_to_state_stage(argv, in_buf, out_buf, len, queue_cur->exec_cksum)) @@ -550,11 +501,7 @@ u8 fuzz_one_original(char** argv) { : havoc_max_mult * 100)) || queue_cur->passed_det) { -#ifdef USE_PYTHON - goto python_stage; -#else - goto havoc_stage; -#endif + goto custom_mutator_stage; } @@ -563,11 +510,7 @@ u8 fuzz_one_original(char** argv) { if (master_max && (queue_cur->exec_cksum % master_max) != master_id - 1) { -#ifdef USE_PYTHON - goto python_stage; -#else - goto havoc_stage; -#endif + goto custom_mutator_stage; } @@ -1582,24 +1525,25 @@ skip_extras: if (!queue_cur->passed_det) mark_as_det_done(queue_cur); -#ifdef USE_PYTHON -python_stage: - /********************************** - * EXTERNAL MUTATORS (Python API) * - **********************************/ +custom_mutator_stage: + /******************* + * CUSTOM MUTATORS * + *******************/ - if (!py_module) goto havoc_stage; + if (!mutator) goto havoc_stage; + if (!mutator->afl_custom_fuzz) goto havoc_stage; - stage_name = "python"; - stage_short = "python"; + stage_name = "custom mutator"; + stage_short = "custom"; stage_max = HAVOC_CYCLES * perf_score / havoc_div / 100; + stage_val_type = STAGE_VAL_NONE; if (stage_max < HAVOC_MIN) stage_max = HAVOC_MIN; - orig_hit_cnt = queued_paths + unique_crashes; + const u32 max_seed_size = 4096 * 4096; + u8* mutated_buf = ck_alloc(max_seed_size); - char* retbuf = NULL; - size_t retlen = 0; + orig_hit_cnt = queued_paths + unique_crashes; for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) { @@ -1646,26 +1590,24 @@ python_stage: ck_read(fd, new_buf, target->len, target->fname); close(fd); - fuzz_py(out_buf, len, new_buf, target->len, &retbuf, &retlen); + size_t mutated_size = mutator->afl_custom_fuzz(out_buf, len, + new_buf, target->len, + mutated_buf, max_seed_size); ck_free(new_buf); - if (retbuf) { + if (mutated_size > 0) { - if (!retlen) goto abandon_entry; + out_buf = ck_realloc(out_buf, mutated_size); + memcpy(out_buf, mutated_buf, mutated_size); - if (common_fuzz_stuff(argv, retbuf, retlen)) { + if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) { - free(retbuf); + ck_free(mutated_buf); goto abandon_entry; } - /* Reset retbuf/retlen */ - free(retbuf); - retbuf = NULL; - retlen = 0; - /* If we're finding new stuff, let's run for a bit longer, limits permitting. */ @@ -1686,12 +1628,13 @@ python_stage: } + ck_free(mutated_buf); new_hit_cnt = queued_paths + unique_crashes; - stage_finds[STAGE_PYTHON] += new_hit_cnt - orig_hit_cnt; - stage_cycles[STAGE_PYTHON] += stage_max; + stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt; + stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max; - if (python_only) { + if (custom_only) { /* Skip other stages */ ret_val = 0; @@ -1699,8 +1642,6 @@ python_stage: } -#endif - /**************** * RANDOM HAVOC * ****************/ @@ -2269,11 +2210,10 @@ retry_splicing: out_buf = ck_alloc_nozero(len); memcpy(out_buf, in_buf, len); -#ifdef USE_PYTHON - goto python_stage; -#else - goto havoc_stage; -#endif + goto custom_mutator_stage; + /* ???: While integrating Python module, the author decided to jump to + python stage, but the reason behind this is not clear.*/ + // goto havoc_stage; } diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 42286527..c22e4402 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -28,195 +28,244 @@ /* Python stuff */ #ifdef USE_PYTHON -int init_py() { +int init_py_module(u8* module_name) { - Py_Initialize(); - u8* module_name = getenv("AFL_PYTHON_MODULE"); - - if (module_name) { + if (!module_name) return 1; - if (limit_time_sig) - FATAL( - "MOpt and Python mutator are mutually exclusive. We accept pull " - "requests that integrates MOpt with the optional mutators " - "(custom/radamsa/redquenn/...)."); + Py_Initialize(); #if PY_MAJOR_VERSION >= 3 - PyObject* py_name = PyUnicode_FromString(module_name); + PyObject* py_name = PyUnicode_FromString(module_name); #else - PyObject* py_name = PyString_FromString(module_name); + PyObject* py_name = PyString_FromString(module_name); #endif - py_module = PyImport_Import(py_name); - Py_DECREF(py_name); + py_module = PyImport_Import(py_name); + Py_DECREF(py_name); + + if (py_module != NULL) { - if (py_module != NULL) { + u8 py_notrim = 0, py_idx; + py_functions[PY_FUNC_INIT] = PyObject_GetAttrString(py_module, "init"); + py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "fuzz"); + py_functions[PY_FUNC_PRE_SAVE] = + PyObject_GetAttrString(py_module, "pre_save"); + py_functions[PY_FUNC_INIT_TRIM] = + PyObject_GetAttrString(py_module, "init_trim"); + py_functions[PY_FUNC_POST_TRIM] = + PyObject_GetAttrString(py_module, "post_trim"); + py_functions[PY_FUNC_TRIM] = PyObject_GetAttrString(py_module, "trim"); - u8 py_notrim = 0, py_idx; - py_functions[PY_FUNC_INIT] = PyObject_GetAttrString(py_module, "init"); - py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "fuzz"); - py_functions[PY_FUNC_INIT_TRIM] = - PyObject_GetAttrString(py_module, "init_trim"); - py_functions[PY_FUNC_POST_TRIM] = - PyObject_GetAttrString(py_module, "post_trim"); - py_functions[PY_FUNC_TRIM] = PyObject_GetAttrString(py_module, "trim"); + for (py_idx = 0; py_idx < PY_FUNC_COUNT; ++py_idx) { - for (py_idx = 0; py_idx < PY_FUNC_COUNT; ++py_idx) { + if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { - if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { + if (py_idx == PY_FUNC_PRE_SAVE) { - if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { + // Implenting the pre_save API is optional for now + if (PyErr_Occurred()) PyErr_Print(); - // Implementing the trim API is optional for now - if (PyErr_Occurred()) PyErr_Print(); - py_notrim = 1; + } else if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { - } else { + // Implementing the trim API is optional for now + if (PyErr_Occurred()) PyErr_Print(); + py_notrim = 1; - if (PyErr_Occurred()) PyErr_Print(); - fprintf(stderr, - "Cannot find/call function with index %d in external " - "Python module.\n", - py_idx); - return 1; + } else { - } + if (PyErr_Occurred()) PyErr_Print(); + fprintf(stderr, + "Cannot find/call function with index %d in external " + "Python module.\n", + py_idx); + return 1; } } - if (py_notrim) { + } - py_functions[PY_FUNC_INIT_TRIM] = NULL; - py_functions[PY_FUNC_POST_TRIM] = NULL; - py_functions[PY_FUNC_TRIM] = NULL; - WARNF( - "Python module does not implement trim API, standard trimming will " - "be used."); + if (py_notrim) { - } + py_functions[PY_FUNC_INIT_TRIM] = NULL; + py_functions[PY_FUNC_POST_TRIM] = NULL; + py_functions[PY_FUNC_TRIM] = NULL; + WARNF( + "Python module does not implement trim API, standard trimming will " + "be used."); + + } + + } else { + + PyErr_Print(); + fprintf(stderr, "Failed to load \"%s\"\n", module_name); + return 1; + + } + + return 0; + +} - PyObject *py_args, *py_value; +void finalize_py_module() { - /* Provide the init function a seed for the Python RNG */ - py_args = PyTuple_New(1); + if (py_module != NULL) { + + u32 i; + for (i = 0; i < PY_FUNC_COUNT; ++i) + Py_XDECREF(py_functions[i]); + + Py_DECREF(py_module); + + } + + Py_Finalize(); + +} + +void init_py(unsigned int seed) { + PyObject *py_args, *py_value; + + /* Provide the init function a seed for the Python RNG */ + py_args = PyTuple_New(1); #if PY_MAJOR_VERSION >= 3 - py_value = PyLong_FromLong(UR(0xFFFFFFFF)); + py_value = PyLong_FromLong(seed); #else - py_value = PyInt_FromLong(UR(0xFFFFFFFF)); + py_value = PyInt_FromLong(seed); #endif - if (!py_value) { + if (!py_value) { + + Py_DECREF(py_args); + fprintf(stderr, "Cannot convert argument\n"); + return; - Py_DECREF(py_args); - fprintf(stderr, "Cannot convert argument\n"); - return 1; + } - } + PyTuple_SetItem(py_args, 0, py_value); - PyTuple_SetItem(py_args, 0, py_value); + py_value = PyObject_CallObject(py_functions[PY_FUNC_INIT], py_args); - py_value = PyObject_CallObject(py_functions[PY_FUNC_INIT], py_args); + Py_DECREF(py_args); - Py_DECREF(py_args); + if (py_value == NULL) { - if (py_value == NULL) { + PyErr_Print(); + fprintf(stderr, "Call failed\n"); + return; - PyErr_Print(); - fprintf(stderr, "Call failed\n"); - return 1; + } +} - } +size_t fuzz_py(u8* buf, size_t buf_size, + u8* add_buf, size_t add_buf_size, + u8* mutated_out, size_t max_size) { - } else { + size_t mutated_size; + PyObject *py_args, *py_value; + py_args = PyTuple_New(3); - PyErr_Print(); - fprintf(stderr, "Failed to load \"%s\"\n", module_name); - return 1; + /* buf */ + py_value = PyByteArray_FromStringAndSize(buf, buf_size); + if (!py_value) { - } + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); } - return 0; + PyTuple_SetItem(py_args, 0, py_value); -} + /* add_buf */ + py_value = PyByteArray_FromStringAndSize(add_buf, add_buf_size); + if (!py_value) { + + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); -void finalize_py() { + } - if (py_module != NULL) { + PyTuple_SetItem(py_args, 1, py_value); - u32 i; - for (i = 0; i < PY_FUNC_COUNT; ++i) - Py_XDECREF(py_functions[i]); + /* max_size */ +#if PY_MAJOR_VERSION >= 3 + py_value = PyLong_FromLong(max_size); +#else + py_value = PyInt_FromLong(max_size); +#endif + if (!py_value) { - Py_DECREF(py_module); + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); } - Py_Finalize(); + PyTuple_SetItem(py_args, 2, py_value); -} + py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args); -void fuzz_py(char* buf, size_t buflen, char* add_buf, size_t add_buflen, - char** ret, size_t* retlen) { + Py_DECREF(py_args); - if (py_module != NULL) { + if (py_value != NULL) { - PyObject *py_args, *py_value; - py_args = PyTuple_New(2); - py_value = PyByteArray_FromStringAndSize(buf, buflen); - if (!py_value) { + mutated_size = PyByteArray_Size(py_value); + memcpy(mutated_out, PyByteArray_AsString(py_value), mutated_size); + Py_DECREF(py_value); + return mutated_size; - Py_DECREF(py_args); - fprintf(stderr, "Cannot convert argument\n"); - return; + } else { - } + PyErr_Print(); + FATAL("Call failed"); - PyTuple_SetItem(py_args, 0, py_value); + } - py_value = PyByteArray_FromStringAndSize(add_buf, add_buflen); - if (!py_value) { +} - Py_DECREF(py_args); - fprintf(stderr, "Cannot convert argument\n"); - return; +size_t pre_save_py(u8* buf, size_t buf_size, u8** out_buf) { - } + size_t out_buf_size; + PyObject *py_args, *py_value; + py_args = PyTuple_New(2); + py_value = PyByteArray_FromStringAndSize(buf, buf_size); + if (!py_value) { - PyTuple_SetItem(py_args, 1, py_value); + Py_DECREF(py_args); + FATAL("Failed to convert arguments"); - py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args); + } - Py_DECREF(py_args); + PyTuple_SetItem(py_args, 0, py_value); - if (py_value != NULL) { + py_value = PyObject_CallObject(py_functions[PY_FUNC_PRE_SAVE], py_args); - *retlen = PyByteArray_Size(py_value); - *ret = malloc(*retlen); - memcpy(*ret, PyByteArray_AsString(py_value), *retlen); - Py_DECREF(py_value); + Py_DECREF(py_args); - } else { + if (py_value != NULL) { - PyErr_Print(); - fprintf(stderr, "Call failed\n"); - return; + out_buf_size = PyByteArray_Size(py_value); + *out_buf = malloc(out_buf_size); + memcpy(*out_buf, PyByteArray_AsString(py_value), out_buf_size); + Py_DECREF(py_value); + return out_buf_size; - } + } else { + + PyErr_Print(); + FATAL("Call failed"); } } -u32 init_trim_py(char* buf, size_t buflen) { +u32 init_trim_py(u8* buf, size_t buf_size) { PyObject *py_args, *py_value; py_args = PyTuple_New(1); - py_value = PyByteArray_FromStringAndSize(buf, buflen); + py_value = PyByteArray_FromStringAndSize(buf, buf_size); if (!py_value) { Py_DECREF(py_args); @@ -248,7 +297,7 @@ u32 init_trim_py(char* buf, size_t buflen) { } -u32 post_trim_py(char success) { +u32 post_trim_py(u8 success) { PyObject *py_args, *py_value; @@ -286,7 +335,7 @@ u32 post_trim_py(char success) { } -void trim_py(char** ret, size_t* retlen) { +void trim_py(u8** out_buf, size_t* out_buf_size) { PyObject *py_args, *py_value; @@ -296,9 +345,9 @@ void trim_py(char** ret, size_t* retlen) { if (py_value != NULL) { - *retlen = PyByteArray_Size(py_value); - *ret = malloc(*retlen); - memcpy(*ret, PyByteArray_AsString(py_value), *retlen); + *out_buf_size = PyByteArray_Size(py_value); + *out_buf = malloc(*out_buf_size); + memcpy(*out_buf, PyByteArray_AsString(py_value), *out_buf_size); Py_DECREF(py_value); } else { @@ -310,126 +359,5 @@ void trim_py(char** ret, size_t* retlen) { } -u8 trim_case_python(char** argv, struct queue_entry* q, u8* in_buf) { - - static u8 tmp[64]; - static u8 clean_trace[MAP_SIZE]; - - u8 needs_write = 0, fault = 0; - u32 trim_exec = 0; - u32 orig_len = q->len; - - stage_name = tmp; - bytes_trim_in += q->len; - - /* Initialize trimming in the Python module */ - stage_cur = 0; - stage_max = init_trim_py(in_buf, q->len); - - if (not_on_tty && debug) - SAYF("[Python Trimming] START: Max %d iterations, %u bytes", stage_max, - q->len); - - while (stage_cur < stage_max) { - - sprintf(tmp, "ptrim %s", DI(trim_exec)); - - u32 cksum; - - char* retbuf = NULL; - size_t retlen = 0; - - trim_py(&retbuf, &retlen); - - if (retlen > orig_len) - FATAL( - "Trimmed data returned by Python module is larger than original " - "data"); - - write_to_testcase(retbuf, retlen); - - fault = run_target(argv, exec_tmout); - ++trim_execs; - - if (stop_soon || fault == FAULT_ERROR) { - - free(retbuf); - goto abort_trimming; - - } - - cksum = hash32(trace_bits, MAP_SIZE, HASH_CONST); - - if (cksum == q->exec_cksum) { - - q->len = retlen; - memcpy(in_buf, retbuf, retlen); - - /* Let's save a clean trace, which will be needed by - update_bitmap_score once we're done with the trimming stuff. */ - - if (!needs_write) { - - needs_write = 1; - memcpy(clean_trace, trace_bits, MAP_SIZE); - - } - - /* Tell the Python module that the trimming was successful */ - stage_cur = post_trim_py(1); - - if (not_on_tty && debug) - SAYF("[Python Trimming] SUCCESS: %d/%d iterations (now at %u bytes)", - stage_cur, stage_max, q->len); - - } else { - - /* Tell the Python module that the trimming was unsuccessful */ - stage_cur = post_trim_py(0); - if (not_on_tty && debug) - SAYF("[Python Trimming] FAILURE: %d/%d iterations", stage_cur, - stage_max); - - } - - free(retbuf); - - /* Since this can be slow, update the screen every now and then. */ - - if (!(trim_exec++ % stats_update_freq)) show_stats(); - - } - - if (not_on_tty && debug) - SAYF("[Python Trimming] DONE: %u bytes -> %u bytes", orig_len, q->len); - - /* If we have made changes to in_buf, we also need to update the on-disk - version of the test case. */ - - if (needs_write) { - - s32 fd; - - unlink(q->fname); /* ignore errors */ - - fd = open(q->fname, O_WRONLY | O_CREAT | O_EXCL, 0600); - - if (fd < 0) PFATAL("Unable to create '%s'", q->fname); - - ck_write(fd, in_buf, q->len, q->fname); - close(fd); - - memcpy(trace_bits, clean_trace, MAP_SIZE); - update_bitmap_score(q); - - } - -abort_trimming: - - bytes_trim_out += q->len; - return fault; - -} - #endif /* USE_PYTHON */ diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 95c2c5d4..12352355 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -309,11 +309,12 @@ void write_to_testcase(void* mem, u32 len) { lseek(fd, 0, SEEK_SET); - if (pre_save_handler) { + if (mutator && mutator->afl_custom_pre_save) { u8* new_data; - size_t new_size = pre_save_handler(mem, len, &new_data); + size_t new_size = mutator->afl_custom_pre_save(mem, len, &new_data); ck_write(fd, new_data, new_size, out_file); + ck_free(new_data); } else { @@ -678,9 +679,8 @@ void sync_fuzzers(char** argv) { u8 trim_case(char** argv, struct queue_entry* q, u8* in_buf) { -#ifdef USE_PYTHON - if (py_functions[PY_FUNC_TRIM]) return trim_case_python(argv, q, in_buf); -#endif + /* Custom mutator trimmer */ + if (mutator->afl_custom_trim) return trim_case_custom(argv, q, in_buf); static u8 tmp[64]; static u8 clean_trace[MAP_SIZE]; diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index c1aa8315..1b763c01 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -655,7 +655,7 @@ void show_stats(void) { } - if (custom_mutator) { + if (mutator) { sprintf(tmp, "%s/%s", DI(stage_finds[STAGE_CUSTOM_MUTATOR]), DI(stage_cycles[STAGE_CUSTOM_MUTATOR])); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 548f029b..2d5a5743 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -155,10 +155,9 @@ static void usage(u8* argv0, int more_help) { "LD_BIND_LAZY: do not set LD_BIND_NOW env var for target\n" "AFL_BENCH_JUST_ONE: run the target just once\n" "AFL_DUMB_FORKSRV: use fork server without feedback from target\n" - "AFL_CUSTOM_MUTATOR_LIBRARY: lib with afl_custom_mutator() to mutate inputs\n" + "AFL_CUSTOM_MUTATOR_LIBRARY: lib with afl_custom_fuzz() to mutate inputs\n" "AFL_CUSTOM_MUTATOR_ONLY: avoid AFL++'s internal mutators\n" "AFL_PYTHON_MODULE: mutate and trim inputs with the specified Python module\n" - "AFL_PYTHON_ONLY: skip AFL++'s own mutators\n" "AFL_DEBUG: extra debugging output for Python mode trimming\n" "AFL_DISABLE_TRIM: disable the trimming of test cases\n" "AFL_NO_UI: switch status screen off\n" @@ -195,7 +194,7 @@ static void usage(u8* argv0, int more_help) { "use \"-hh\".\n\n"); #ifdef USE_PYTHON - SAYF("Compiled with %s module support, see docs/python_mutators.md\n", + SAYF("Compiled with %s module support, see docs/custom_mutator.md\n", (char*)PYTHON_VERSION); #endif @@ -658,11 +657,10 @@ int main(int argc, char** argv, char** envp) { OKF("afl-tmin fork server patch from github.com/nccgroup/TriforceAFL"); OKF("MOpt Mutator from github.com/puppet-meteor/MOpt-AFL"); - if (sync_id && force_deterministic && - (getenv("AFL_CUSTOM_MUTATOR_ONLY") || getenv("AFL_PYTHON_ONLY"))) + if (sync_id && force_deterministic && getenv("AFL_CUSTOM_MUTATOR_ONLY")) WARNF( - "Using -M master with the AFL_..._ONLY mutator options will result in " - "no deterministic mutations being done!"); + "Using -M master with the AFL_CUSTOM_MUTATOR_ONLY mutator options will " + "result in no deterministic mutations being done!"); check_environment_vars(envp); @@ -752,9 +750,9 @@ int main(int argc, char** argv, char** envp) { if (get_afl_env("AFL_FAST_CAL")) fast_cal = 1; if (get_afl_env("AFL_AUTORESUME")) { - + autoresume = 1; - if (in_place_resume) + if (in_place_resume) SAYF("AFL_AUTORESUME has no effect for '-i -'"); } @@ -832,16 +830,6 @@ int main(int argc, char** argv, char** envp) { if (get_afl_env("AFL_DEBUG")) debug = 1; - if (get_afl_env("AFL_PYTHON_ONLY")) { - - /* This ensures we don't proceed to havoc/splice */ - python_only = 1; - - /* Ensure we also skip all deterministic steps */ - skip_deterministic = 1; - - } - if (get_afl_env("AFL_CUSTOM_MUTATOR_ONLY")) { /* This ensures we don't proceed to havoc/splice */ @@ -862,7 +850,6 @@ int main(int argc, char** argv, char** envp) { check_cpu_governor(); setup_post(); - setup_custom_mutator(); setup_shm(dumb_mode); if (!in_bitmap) memset(virgin_bits, 255, MAP_SIZE); @@ -873,12 +860,7 @@ int main(int argc, char** argv, char** envp) { setup_dirs_fds(); -#ifdef USE_PYTHON - if (init_py()) FATAL("Failed to initialize Python module"); -#else - if (getenv("AFL_PYTHON_MODULE")) - FATAL("Your AFL binary was built without Python support"); -#endif + setup_custom_mutator(); setup_cmdline_file(argv + optind); @@ -1156,13 +1138,10 @@ stop_fuzzing: destroy_extras(); ck_free(target_path); ck_free(sync_id); + destroy_custom_mutator(); alloc_report(); -#ifdef USE_PYTHON - finalize_py(); -#endif - OKF("We're done here. Have a nice day!\n"); exit(0); |