diff options
| author | van Hauser <vh@thc.org> | 2022-06-28 14:18:44 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-06-28 14:18:44 +0200 |
| commit | 40947508037b874020c8dd1251359fecaab04b9d (patch) | |
| tree | e512d9b04e55c619adaefb22cd9ed1e33eac1feb | |
| parent | ba3c7bfe40f9b17a691958e3525828385127ad25 (diff) | |
| parent | 92352951d7a8485bd2413009fcd052e85dc398fb (diff) | |
| download | afl++-40947508037b874020c8dd1251359fecaab04b9d.tar.gz | |
Merge pull request #1438 from AFLplusplus/dev4.01c
push to stable
46 files changed, 310 insertions, 1060 deletions
diff --git a/.github/workflows/build_aflplusplus_docker.yaml b/.github/workflows/build_aflplusplus_docker.yaml index fa96da8e..7245a84e 100644 --- a/.github/workflows/build_aflplusplus_docker.yaml +++ b/.github/workflows/build_aflplusplus_docker.yaml @@ -2,29 +2,48 @@ name: Publish Docker Images on: push: - branches: [ stable ] -# paths: -# - Dockerfile + branches: + - stable + - dev + tags: + - '*' jobs: push_to_registry: name: Push Docker images to Dockerhub runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to Dockerhub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Publish aflpp to Registry - uses: docker/build-push-action@v2 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: aflplusplus/aflplusplus:latest + - uses: actions/checkout@master + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Dockerhub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Publish dev as dev to docker.io registry + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: aflplusplus/aflplusplus:${{ github.ref_name }} + if: ${{ github.ref_name == 'dev' }} + - name: Publish stable as stable and latest to docker.io registry + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: aflplusplus/aflplusplus:${{ github.ref_name }},aflplusplus/aflplusplus:latest + if: ${{ github.ref_name == 'stable' }} + - name: Publish tagged release to docker.io registry + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: aflplusplus/aflplusplus:${{ github.ref_name }} + if: ${{ github.ref_type == 'tag' }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 886148df..799b72e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: runs-on: '${{ matrix.os }}' strategy: matrix: - os: [ubuntu-20.04, ubuntu-18.04] + os: [ubuntu-22.04, ubuntu-20.04, ubuntu-18.04] env: AFL_SKIP_CPUFREQ: 1 AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 diff --git a/.github/workflows/rust_custom_mutator.yml b/.github/workflows/rust_custom_mutator.yml index de2b184a..c279439e 100644 --- a/.github/workflows/rust_custom_mutator.yml +++ b/.github/workflows/rust_custom_mutator.yml @@ -15,7 +15,7 @@ jobs: working-directory: custom_mutators/rust strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-22.04, ubuntu-20.04] steps: - uses: actions/checkout@v2 - name: Install Rust Toolchain @@ -97,3 +97,4 @@ utils/persistent_mode/persistent_demo_new utils/persistent_mode/test-instr !coresight_mode !coresight_mode/coresight-trace +vuln_prog
\ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 8ba1c39d..18fda27e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,9 +10,6 @@ [submodule "custom_mutators/gramatron/json-c"] path = custom_mutators/gramatron/json-c url = https://github.com/json-c/json-c -[submodule "utils/optimin/EvalMaxSAT"] - path = utils/optimin/EvalMaxSAT - url = https://github.com/FlorentAvellaneda/EvalMaxSAT [submodule "coresight_mode/patchelf"] path = coresight_mode/patchelf url = https://github.com/NixOS/patchelf.git @@ -76,6 +76,7 @@ cc_binary { srcs: [ "src/afl-fuzz*.c", "src/afl-common.c", + "src/afl-forkserver.c", "src/afl-sharedmem.c", "src/afl-forkserver.c", "src/afl-performance.c", diff --git a/GNUmakefile b/GNUmakefile index 072bd09d..42d48b68 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -696,6 +696,7 @@ install: all $(MANPAGES) @rm -f $${DESTDIR}$(BIN_PATH)/afl-plot.sh @rm -f $${DESTDIR}$(BIN_PATH)/afl-as @rm -f $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-32.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-64.o $${DESTDIR}$(HELPER_PATH)/afl-gcc-rt.o + @for i in afl-llvm-dict2file.so afl-llvm-lto-instrumentlist.so afl-llvm-pass.so cmplog-instructions-pass.so cmplog-routines-pass.so cmplog-switches-pass.so compare-transform-pass.so libcompcov.so libdislocator.so libnyx.so libqasan.so libtokencap.so SanitizerCoverageLTO.so SanitizerCoveragePCGUARD.so split-compares-pass.so split-switches-pass.so; do echo rm -fv $${DESTDIR}$(HELPER_PATH)/$${i}; done install -m 755 $(PROGS) $(SH_PROGS) $${DESTDIR}$(BIN_PATH) @if [ -f afl-qemu-trace ]; then install -m 755 afl-qemu-trace $${DESTDIR}$(BIN_PATH); fi @if [ -f utils/plot_ui/afl-plot-ui ]; then install -m 755 utils/plot_ui/afl-plot-ui $${DESTDIR}$(BIN_PATH); fi @@ -2,9 +2,9 @@ <img align="right" src="https://raw.githubusercontent.com/AFLplusplus/Website/master/static/aflpp_bg.svg" alt="AFL++ logo" width="250" heigh="250"> -Release version: [4.00c](https://github.com/AFLplusplus/AFLplusplus/releases) +Release version: [4.01c](https://github.com/AFLplusplus/AFLplusplus/releases) -GitHub version: 4.01a +GitHub version: 4.02a Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) @@ -50,17 +50,20 @@ Here is some information to get you started: ## Building and installing AFL++ To have AFL++ easily available with everything compiled, pull the image directly -from the Docker Hub (available for x86_64 and arm64): +from the Docker Hub (available for both x86_64 and arm64): ```shell docker pull aflplusplus/aflplusplus docker run -ti -v /location/of/your/target:/src aflplusplus/aflplusplus ``` -This image is automatically generated when a push to the stable repo happens +This image is automatically published when a push to the stable branch happens (see [branches](#branches)). If you use the command above, you will find your target source code in `/src` in the container. +Note: you can also pull `aflplusplus/aflplusplus:dev` which is the most current +development state of AFL++. + To build AFL++ yourself - *which we recommend* - continue at [docs/INSTALL.md](docs/INSTALL.md). @@ -2,6 +2,7 @@ ## Should + - makefiles should show provide a build summary success/failure - better documentation for custom mutators - better autodetection of shifting runtime timeout values - Update afl->pending_not_fuzzed for MOpt @@ -135,6 +135,12 @@ function exists_and_is_executable(binarypath) { } BEGIN { + if (0 != system( "test -t 1")) { + redirected = 1 + } else { + redirected = 0 + } + print "corpus minimization tool for afl++ (awk version)\n" # defaults @@ -463,7 +469,8 @@ BEGIN { while (cur < in_count) { fn = infilesSmallToBig[cur] ++cur - printf "\r Processing file "cur"/"in_count + if (redirected == 0) { printf "\r Processing file "cur"/"in_count } + else { print " Processing file "cur"/"in_count } # create path for the trace file from afl-showmap tracefile_path = trace_dir"/"fn # gather all keys, and count them @@ -502,7 +509,9 @@ BEGIN { key = field[nrFields] ++tcnt; - printf "\r Processing tuple "tcnt"/"tuple_count" with count "key_count[key]"..." + if (redirected == 0) { printf "\r Processing tuple "tcnt"/"tuple_count" with count "key_count[key]"..." } + else { print " Processing tuple "tcnt"/"tuple_count" with count "key_count[key]"..." } + if (key in keyAlreadyKnown) { continue } diff --git a/afl-system-config b/afl-system-config index ef343704..f482e4fb 100755 --- a/afl-system-config +++ b/afl-system-config @@ -118,7 +118,7 @@ if [ "$PLATFORM" = "Darwin" ] ; then sudo launchctl unload -w ${SL}/LaunchDaemons/${PL}.Root.plist >/dev/null 2>&1 echo fi - echo It is recommended to disable System Integration Protection for increased performance. + echo It is recommended to disable System Integrity Protection for increased performance. echo DONE=1 fi diff --git a/coresight_mode/patchelf b/coresight_mode/patchelf -Subproject 7ec8edbe094ee13c91dadca191f92b9dfac8c0f +Subproject be0cc30a59b2755844bcd48823f6fbc8d97b93a diff --git a/custom_mutators/gramatron/json-c b/custom_mutators/gramatron/json-c -Subproject af8dd4a307e7b837f9fa2959549548ace4afe08 +Subproject 11546bfd07a575c47416924cb98de3d33a4e642 diff --git a/custom_mutators/grammar_mutator/build_grammar_mutator.sh b/custom_mutators/grammar_mutator/build_grammar_mutator.sh index 15b8b1db..e8594ba3 100755 --- a/custom_mutators/grammar_mutator/build_grammar_mutator.sh +++ b/custom_mutators/grammar_mutator/build_grammar_mutator.sh @@ -119,7 +119,7 @@ else } fi -test -d grammar_mutator/.git || { echo "[-] not checked out, please install git or check your internet connection." ; exit 1 ; } +test -f grammar_mutator/.git || { echo "[-] not checked out, please install git or check your internet connection." ; exit 1 ; } echo "[+] Got grammar mutator." cd "grammar_mutator" || exit 1 diff --git a/docs/Changelog.md b/docs/Changelog.md index b18bf30f..737df7fa 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -8,8 +8,8 @@ Want to stay in the loop on major new features? Join our mailing list by sending a mail to <afl-users+subscribe@googlegroups.com>. -### Version ++4.01a (dev) - - fix */build_...sh scripts to work outside of git +### Version ++4.01c (release) + - fixed */build_...sh scripts to work outside of git - new custom_mutator: libafl with token fuzzing :) - afl-fuzz: - when you just want to compile once and set CMPLOG, then just @@ -17,6 +17,8 @@ sending a mail to <afl-users+subscribe@googlegroups.com>. CMPLOG. - new commandline options -g/G to set min/max length of generated fuzz inputs + - you can set the time for syncing to other fuzzer now with + AFL_SYNC_TIME - reintroduced AFL_PERSISTENT and AFL_DEFER_FORKSRV to allow persistent mode and manual forkserver support if these are not in the target binary (e.g. are in a shared library) @@ -28,6 +30,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>. kept), unless AFL_KEEP_TIMEOUTS are set - AFL never implemented auto token inserts (but user token inserts, user token overwrite and auto token overwrite), added now! + - fixed a mutation type in havoc mode - Mopt fix to always select the correct algorithm - fix effector map calculation (deterministic mode) - fix custom mutator post_process functionality @@ -41,6 +44,9 @@ sending a mail to <afl-users+subscribe@googlegroups.com>. - update to new frida release, handles now c++ throw/catch - unicorn_mode: - update unicorn engine, fix C example + - utils: + - removed optimin because it looses coverage due to a bug and is + unmaintained :-( ### Version ++4.00c (release) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 01343b7f..41ec8561 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -8,16 +8,22 @@ hence afl-clang-lto is available) or just pull directly from the Docker Hub (for x86_64 and arm64): ```shell -docker pull aflplusplus/aflplusplus +docker pull aflplusplus/aflplusplus: docker run -ti -v /location/of/your/target:/src aflplusplus/aflplusplus ``` -This image is automatically generated when a push to the stable repo happens. +This image is automatically generated when a push to the stable branch happens. You will find your target source code in `/src` in the container. +Note: you can also pull `aflplusplus/aflplusplus:dev` which is the most current +development state of AFL++. + If you want to build AFL++ yourself, you have many options. The easiest choice is to build and install everything: +NOTE: depending on your Debian/Ubuntu/Kali/... version replease `-12` with +whatever llvm version is available! + ```shell sudo apt-get update sudo apt-get install -y build-essential python3-dev automake cmake git flex bison libglib2.0-dev libpixman-1-dev python3-setuptools diff --git a/docs/env_variables.md b/docs/env_variables.md index fe9c6e07..0598a809 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -517,6 +517,10 @@ checks or alter some of the more exotic semantics of the tool: (empty/non present) will add no tags to the metrics. For more information, see [rpc_statsd.md](rpc_statsd.md). + - `AFL_SYNC_TIME` allows you to specify a different minimal time (in minutes) + between fuzzing instances synchronization. Default sync time is 30 minutes, + note that time is halved for -M main nodes. + - Setting `AFL_TARGET_ENV` causes AFL++ to set extra environment variables for the target binary. Example: `AFL_TARGET_ENV="VAR1=1 VAR2='a b c'" afl-fuzz ... `. This exists mostly for things like `LD_LIBRARY_PATH` but it would @@ -615,6 +619,10 @@ The QEMU wrapper used to instrument binary-only code supports several settings: emulation" variables (e.g., `QEMU_STACK_SIZE`), but there should be no reason to touch them. + - Normally a `README.txt` is written to the `crashes/` directory when a first + crash is found. Setting `AFL_NO_CRASH_README` will prevent this. Useful when + counting crashes based on a file count in that directory. + ## 7) Settings for afl-frida-trace The FRIDA wrapper used to instrument binary-only code supports many of the same diff --git a/docs/tutorials.md b/docs/tutorials.md index 64d2b376..477ff98b 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -1,5 +1,9 @@ # Tutorials +If you are a total newbie, try this guide: + +* [https://github.com/alex-maleno/Fuzzing-Module](https://github.com/alex-maleno/Fuzzing-Module) + Here are some good write-ups to show how to effectively use AFL++: * [https://aflplus.plus/docs/tutorials/libxml2_tutorial/](https://aflplus.plus/docs/tutorials/libxml2_tutorial/) @@ -17,7 +21,7 @@ training, then we can highly recommend the following: * [https://github.com/antonio-morales/Fuzzing101](https://github.com/antonio-morales/Fuzzing101) If you are interested in fuzzing structured data (where you define what the -structure is), these links have you covered: +structure is), these links have you covered (some are outdated though): * libprotobuf for AFL++: [https://github.com/P1umer/AFLplusplus-protobuf-mutator](https://github.com/P1umer/AFLplusplus-protobuf-mutator) diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 9992e841..ce42a107 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -386,7 +386,7 @@ typedef struct afl_env_vars { afl_bench_until_crash, afl_debug_child, afl_autoresume, afl_cal_fast, afl_cycle_schedules, afl_expand_havoc, afl_statsd, afl_cmplog_only_new, afl_exit_on_seed_issues, afl_try_affinity, afl_ignore_problems, - afl_keep_timeouts, afl_pizza_mode; + afl_keep_timeouts, afl_pizza_mode, afl_no_crash_readme; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_preload, @@ -577,7 +577,8 @@ typedef struct afl_state { last_find_time, /* Time for most recent path (ms) */ last_crash_time, /* Time for most recent crash (ms) */ last_hang_time, /* Time for most recent hang (ms) */ - exit_on_time; /* Delay to exit if no new paths */ + exit_on_time, /* Delay to exit if no new paths */ + sync_time; /* Sync time (ms) */ u32 slowest_exec_ms, /* Slowest testcase non hang in ms */ subseq_tmouts; /* Number of timeouts in a row */ diff --git a/include/config.h b/include/config.h index 9fc92b06..aefea9cd 100644 --- a/include/config.h +++ b/include/config.h @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.01a" +#define VERSION "++4.01c" /****************************************************** * * diff --git a/include/envs.h b/include/envs.h index 25b792fa..9b8917f9 100644 --- a/include/envs.h +++ b/include/envs.h @@ -159,6 +159,7 @@ static char *afl_environment_variables[] = { "AFL_NO_COLOUR", #endif "AFL_NO_CPU_RED", + "AFL_NO_CRASH_README", "AFL_NO_FORKSRV", "AFL_NO_UI", "AFL_NO_PYTHON", @@ -206,6 +207,7 @@ static char *afl_environment_variables[] = { "AFL_STATSD_HOST", "AFL_STATSD_PORT", "AFL_STATSD_TAGS_FLAVOR", + "AFL_SYNC_TIME", "AFL_TESTCACHE_SIZE", "AFL_TESTCACHE_ENTRIES", "AFL_TMIN_EXACT", diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index db7ac7b0..f3a16e95 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -327,6 +327,41 @@ static void __afl_map_shm(void) { } + if (!id_str && __afl_area_ptr_dummy == __afl_area_initial) { + + u32 val = 0; + u8 *ptr; + + if ((ptr = getenv("AFL_MAP_SIZE")) != NULL) val = atoi(ptr); + + if (val > MAP_INITIAL_SIZE) { + + __afl_map_size = val; + __afl_area_ptr_dummy = malloc(__afl_map_size); + if (!__afl_area_ptr_dummy) { + + fprintf(stderr, + "Error: AFL++ could not aquire %u bytes of memory, exiting!\n", + __afl_map_size); + exit(-1); + + } + + } else { + + __afl_map_size = MAP_INITIAL_SIZE; + + } + + if (__afl_debug) { + + fprintf(stderr, "DEBUG: (0) init map size is %u to %p\n", __afl_map_size, + __afl_area_ptr_dummy); + + } + + } + /* If we're running under AFL, attach to the appropriate region, replacing the early-stage __afl_area_initial region that is needed to allow some really hacky .init code to work correctly in projects such as OpenSSL. */ @@ -465,18 +500,26 @@ static void __afl_map_shm(void) { } - } else if (_is_sancov && __afl_area_ptr != __afl_area_initial) { - - free(__afl_area_ptr); - __afl_area_ptr = NULL; + } else if (__afl_final_loc > __afl_map_size) { - if (__afl_final_loc > MAP_INITIAL_SIZE) { + if (__afl_area_initial != __afl_area_ptr_dummy) { - __afl_area_ptr = (u8 *)malloc(__afl_final_loc); + free(__afl_area_ptr_dummy); } - if (!__afl_area_ptr) { __afl_area_ptr = __afl_area_ptr_dummy; } + __afl_area_ptr_dummy = (u8 *)malloc(__afl_final_loc); + __afl_area_ptr = __afl_area_ptr_dummy; + __afl_map_size = __afl_final_loc; + + if (!__afl_area_ptr_dummy) { + + fprintf(stderr, + "Error: AFL++ could not aquire %u bytes of memory, exiting!\n", + __afl_final_loc); + exit(-1); + + } } @@ -487,7 +530,7 @@ static void __afl_map_shm(void) { fprintf(stderr, "DEBUG: (2) id_str %s, __afl_area_ptr %p, __afl_area_initial %p, " "__afl_area_ptr_dummy %p, __afl_map_addr 0x%llx, MAP_SIZE " - "%u, __afl_final_loc %u, __afl_map_size %u," + "%u, __afl_final_loc %u, __afl_map_size %u, " "max_size_forkserver %u/0x%x\n", id_str == NULL ? "<null>" : id_str, __afl_area_ptr, __afl_area_initial, __afl_area_ptr_dummy, __afl_map_addr, MAP_SIZE, diff --git a/instrumentation/afl-llvm-common.cc b/instrumentation/afl-llvm-common.cc index 9483da83..5fcf27fb 100644 --- a/instrumentation/afl-llvm-common.cc +++ b/instrumentation/afl-llvm-common.cc @@ -291,7 +291,7 @@ void scanForDangerousFunctions(llvm::Module *M) { StringRef r_name = cast<Function>(r->getOperand(0))->getName(); if (!be_quiet) fprintf(stderr, - "Info: Found an ifunc with name %s that points to resolver " + "Note: Found an ifunc with name %s that points to resolver " "function %s, we will not instrument this, putting it into the " "block list.\n", ifunc_name.str().c_str(), r_name.str().c_str()); @@ -329,7 +329,7 @@ void scanForDangerousFunctions(llvm::Module *M) { if (!be_quiet) fprintf(stderr, - "Info: Found constructor function %s with prio " + "Note: Found constructor function %s with prio " "%u, we will not instrument this, putting it into a " "block list.\n", F->getName().str().c_str(), Priority); diff --git a/instrumentation/COPYING3 b/instrumentation/gcc_plugin.COPYING3 index 94a9ed02..b0a36be4 100644 --- a/instrumentation/COPYING3 +++ b/instrumentation/gcc_plugin.COPYING3 @@ -1,3 +1,8 @@ +NOTE: +This license applies only to the gcc_plugin code "afl-gcc-pass.so.cc" as +gcc is GPL3 too. + + GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 diff --git a/src/afl-cc.c b/src/afl-cc.c index 2667ae28..4a56169f 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -396,7 +396,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { snprintf(llvm_fullpath, sizeof(llvm_fullpath), "%s/clang", LLVM_BINDIR); else - snprintf(llvm_fullpath, sizeof(llvm_fullpath), CLANG_BIN); + snprintf(llvm_fullpath, sizeof(llvm_fullpath), "%s", CLANG_BIN); alt_cc = llvm_fullpath; } diff --git a/src/afl-common.c b/src/afl-common.c index eca7d272..7f482e7d 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -25,8 +25,12 @@ #include <stdlib.h> #include <stdio.h> -#define _GNU_SOURCE -#define __USE_GNU +#ifndef _GNU_SOURCE + #define _GNU_SOURCE +#endif +#ifndef __USE_GNU + #define __USE_GNU +#endif #include <string.h> #include <strings.h> #include <math.h> @@ -715,17 +719,23 @@ char *get_afl_env(char *env) { char *val; - if ((val = getenv(env)) != NULL) { + if ((val = getenv(env))) { - if (!be_quiet) { + if (*val) { + + if (!be_quiet) { + + OKF("Enabled environment variable %s with value %s", env, val); + + } - OKF("Loaded environment variable %s with value %s", env, val); + return val; } } - return val; + return NULL; } diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 26e70d81..089f7bb5 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -720,7 +720,12 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { } - if (unlikely(!afl->saved_crashes)) { write_crash_readme(afl); } + if (unlikely(!afl->saved_crashes) && + (afl->afl_env.afl_no_crash_readme != 1)) { + + write_crash_readme(afl); + + } #ifndef SIMPLE_FILES diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 19f41ebe..ef80524f 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -2585,7 +2585,7 @@ havoc_stage: snprintf(afl->m_tmp, sizeof(afl->m_tmp), " SUBBYTE_"); strcat(afl->mutation, afl->m_tmp); #endif - out_buf[rand_below(afl, temp_len)]++; + out_buf[rand_below(afl, temp_len)]--; break; } diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 65501c8c..0231d2cd 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -28,6 +28,36 @@ /* Python stuff */ #ifdef USE_PYTHON +// Tries to cast a python bytearray or bytes to a char ptr +static inline bool py_bytes(PyObject *py_value, /* out */ char **bytes, + /* out */ size_t *size) { + + if (!py_value) { return false; } + + *bytes = PyByteArray_AsString(py_value); + if (*bytes) { + + // we got a bytearray + *size = PyByteArray_Size(py_value); + + } else { + + *bytes = PyBytes_AsString(py_value); + if (!*bytes) { + + // No valid type returned. + return false; + + } + + *size = PyBytes_Size(py_value); + + } + + return true; + +} + static void *unsupported(afl_state_t *afl, unsigned int seed) { (void)afl; @@ -93,12 +123,22 @@ static size_t fuzz_py(void *py_mutator, u8 *buf, size_t buf_size, u8 **out_buf, if (py_value != NULL) { - mutated_size = PyByteArray_Size(py_value); + char *bytes; + if (!py_bytes(py_value, &bytes, &mutated_size)) { + + FATAL("Python mutator fuzz() should return a bytearray or bytes"); + + } + + if (mutated_size) { + + *out_buf = afl_realloc(BUF_PARAMS(fuzz), mutated_size); + if (unlikely(!*out_buf)) { PFATAL("alloc"); } + + memcpy(*out_buf, bytes, mutated_size); - *out_buf = afl_realloc(BUF_PARAMS(fuzz), mutated_size); - if (unlikely(!*out_buf)) { PFATAL("alloc"); } + } - memcpy(*out_buf, PyByteArray_AsString(py_value), mutated_size); Py_DECREF(py_value); return mutated_size; @@ -625,7 +665,7 @@ s32 post_trim_py(void *py_mutator, u8 success) { size_t trim_py(void *py_mutator, u8 **out_buf) { PyObject *py_args, *py_value; - size_t ret; + size_t trimmed_size; py_args = PyTuple_New(0); py_value = PyObject_CallObject( @@ -634,10 +674,21 @@ size_t trim_py(void *py_mutator, u8 **out_buf) { if (py_value != NULL) { - ret = PyByteArray_Size(py_value); - *out_buf = afl_realloc(BUF_PARAMS(trim), ret); - if (unlikely(!*out_buf)) { PFATAL("alloc"); } - memcpy(*out_buf, PyByteArray_AsString(py_value), ret); + char *bytes; + if (!py_bytes(py_value, &bytes, &trimmed_size)) { + + FATAL("Python mutator fuzz() should return a bytearray"); + + } + + if (trimmed_size) { + + *out_buf = afl_realloc(BUF_PARAMS(trim), trimmed_size); + if (unlikely(!*out_buf)) { PFATAL("alloc"); } + memcpy(*out_buf, bytes, trimmed_size); + + } + Py_DECREF(py_value); } else { @@ -647,7 +698,7 @@ size_t trim_py(void *py_mutator, u8 **out_buf) { } - return ret; + return trimmed_size; } @@ -692,7 +743,13 @@ size_t havoc_mutation_py(void *py_mutator, u8 *buf, size_t buf_size, if (py_value != NULL) { - mutated_size = PyByteArray_Size(py_value); + char *bytes; + if (!py_bytes(py_value, &bytes, &mutated_size)) { + + FATAL("Python mutator fuzz() should return a bytearray"); + + } + if (mutated_size <= buf_size) { /* We reuse the input buf here. */ @@ -706,7 +763,7 @@ size_t havoc_mutation_py(void *py_mutator, u8 *buf, size_t buf_size, } - memcpy(*out_buf, PyByteArray_AsString(py_value), mutated_size); + if (mutated_size) { memcpy(*out_buf, bytes, mutated_size); } Py_DECREF(py_value); return mutated_size; @@ -762,7 +819,17 @@ const char *introspection_py(void *py_mutator) { } else { - return PyByteArray_AsString(py_value); + char * ret; + size_t len; + if (!py_bytes(py_value, &ret, &len)) { + + FATAL( + "Python mutator introspection call returned illegal type (expected " + "bytes or bytearray)"); + + } + + return ret; } diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 09e773f0..5703a66a 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -130,11 +130,7 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { } - if (new_mem != *mem) { - - *mem = new_mem; - - } + if (new_mem != *mem) { *mem = new_mem; } /* everything as planned. use the potentially new data. */ afl_fsrv_write_to_testcase(&afl->fsrv, *mem, new_size); diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 98217438..cc4138ae 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -101,6 +101,7 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->stats_update_freq = 1; afl->stats_avg_exec = 0; afl->skip_deterministic = 1; + afl->sync_time = SYNC_TIME; afl->cmplog_lvl = 2; afl->min_length = 1; afl->max_length = MAX_FILE; @@ -509,6 +510,14 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_pizza_mode = atoi((u8 *)get_afl_env(afl_environment_variables[i])); + + } else if (!strncmp(env, "AFL_NO_CRASH_README", + + afl_environment_variable_len)) { + + afl->afl_env.afl_no_crash_readme = + atoi((u8 *)get_afl_env(afl_environment_variables[i])); + if (afl->afl_env.afl_pizza_mode == 0) { afl->afl_env.afl_pizza_mode = 1; @@ -519,6 +528,24 @@ void read_afl_environment(afl_state_t *afl, char **envp) { } + } else if (!strncmp(env, "AFL_SYNC_TIME", + + afl_environment_variable_len)) { + + int time = atoi((u8 *)get_afl_env(afl_environment_variables[i])); + if (time > 0) { + + afl->sync_time = time * (60 * 1000LL); + + } else { + + WARNF( + "incorrect value for AFL_SYNC_TIME environment variable, " + "used default value %lld instead.", + afl->sync_time / 60 / 1000); + + } + } } else { diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 5b237748..3e034b83 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -59,7 +59,7 @@ void write_setup_file(afl_state_t *afl, u32 argc, char **argv) { if (i) fprintf(f, " "); #ifdef __ANDROID__ - if (memchr(argv[i], '\'', sizeof(argv[i]))) { + if (memchr(argv[i], '\'', strlen(argv[i]))) { #else if (index(argv[i], '\'')) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index c5ab364a..b23cef37 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -295,6 +295,8 @@ static void usage(u8 *argv0, int more_help) { "AFL_STATSD_TAGS_FLAVOR: set statsd tags format (default: disable tags)\n" " Supported formats are: 'dogstatsd', 'librato',\n" " 'signalfx' and 'influxdb'\n" + "AFL_SYNC_TIME: sync time between fuzzing instances (in minutes)\n" + "AFL_NO_CRASH_README: do not create a README in the crashes directory\n" "AFL_TESTCACHE_SIZE: use a cache for testcases, improves performance (in MB)\n" "AFL_TMPDIR: directory to use for input file generation (ramdisk recommended)\n" "AFL_EARLY_FORKSERVER: force an early forkserver in an afl-clang-fast/\n" @@ -2511,7 +2513,7 @@ int main(int argc, char **argv_orig, char **envp) { if (unlikely(afl->is_main_node)) { if (unlikely(get_cur_time() > - (SYNC_TIME >> 1) + afl->last_sync_time)) { + (afl->sync_time >> 1) + afl->last_sync_time)) { if (!(sync_interval_cnt++ % (SYNC_INTERVAL / 3))) { @@ -2523,7 +2525,7 @@ int main(int argc, char **argv_orig, char **envp) { } else { - if (unlikely(get_cur_time() > SYNC_TIME + afl->last_sync_time)) { + if (unlikely(get_cur_time() > afl->sync_time + afl->last_sync_time)) { if (!(sync_interval_cnt++ % SYNC_INTERVAL)) { sync_fuzzers(afl); } diff --git a/test/test-frida-mode.sh b/test/test-frida-mode.sh index 59b8e307..9e1f756d 100755 --- a/test/test-frida-mode.sh +++ b/test/test-frida-mode.sh @@ -62,7 +62,7 @@ test -e ../afl-frida-trace.so && { #else #fi export AFL_FRIDA_PERSISTENT_ADDR=0x`nm test-instr | grep -Ei "T _main|T main" | awk '{print $1}'` - $ECHO "Info: AFL_FRIDA_PERSISTENT_ADDR=$AFL_FRIDA_PERSISTENT_ADDR <= $(nm test-instr | grep "T main" | awk '{print $1}')" + $ECHO "Note: AFL_FRIDA_PERSISTENT_ADDR=$AFL_FRIDA_PERSISTENT_ADDR <= $(nm test-instr | grep "T main" | awk '{print $1}')" env|grep AFL_|sort file test-instr export AFL_DEBUG_CHILD=1 diff --git a/unicorn_mode/UNICORNAFL_VERSION b/unicorn_mode/UNICORNAFL_VERSION index 77fc69b5..5e7234c6 100644 --- a/unicorn_mode/UNICORNAFL_VERSION +++ b/unicorn_mode/UNICORNAFL_VERSION @@ -1 +1 @@ -c3e15a7d +06796154996fef2d92ccd172181ee0cdf3631959 diff --git a/unicorn_mode/unicornafl b/unicorn_mode/unicornafl -Subproject c3e15a7d44101ff288abe114b7954ce6cfa070b +Subproject 06796154996fef2d92ccd172181ee0cdf363195 diff --git a/utils/README.md b/utils/README.md index debc86e8..62d79193 100644 --- a/utils/README.md +++ b/utils/README.md @@ -56,8 +56,6 @@ Here's a quick overview of the stuff you can find in this directory: - libpng_no_checksum - a sample patch for removing CRC checks in libpng. - - optimin - An optimal corpus minimizer. - - persistent_mode - an example of how to use the LLVM persistent process mode to speed up certain fuzzing jobs. diff --git a/utils/libdislocator/libdislocator.so.c b/utils/libdislocator/libdislocator.so.c index bd08a678..c821a8f7 100644 --- a/utils/libdislocator/libdislocator.so.c +++ b/utils/libdislocator/libdislocator.so.c @@ -505,7 +505,10 @@ void *reallocarray(void *ptr, size_t elem_len, size_t elem_cnt) { } -#if !defined(__ANDROID__) +#if defined(__APPLE__) +size_t malloc_size(const void *ptr) { + +#elif !defined(__ANDROID__) size_t malloc_usable_size(void *ptr) { #else @@ -517,6 +520,15 @@ size_t malloc_usable_size(const void *ptr) { } +#if defined(__APPLE__) +size_t malloc_good_size(size_t len) { + + return (len & ~(ALLOC_ALIGN_SIZE - 1)) + ALLOC_ALIGN_SIZE; + +} + +#endif + __attribute__((constructor)) void __dislocator_init(void) { char *tmp = getenv("AFL_LD_LIMIT_MB"); diff --git a/utils/optimin/.gitignore b/utils/optimin/.gitignore deleted file mode 100644 index 46f42f8f..00000000 --- a/utils/optimin/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -CMakeLists.txt.user -CMakeCache.txt -CMakeFiles -CMakeScripts -Testing -Makefile -cmake_install.cmake -install_manifest.txt -compile_commands.json -CTestTestfile.cmake -_deps diff --git a/utils/optimin/CMakeLists.txt b/utils/optimin/CMakeLists.txt deleted file mode 100644 index b45dd004..00000000 --- a/utils/optimin/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -cmake_minimum_required(VERSION 3.10) - -project(optimin - LANGUAGES CXX - DESCRIPTION "MaxSAT-based fuzzing corpus minimizer" -) - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra") - -# Add LLVM -find_package(LLVM REQUIRED CONFIG) -message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") - -include_directories(${LLVM_INCLUDE_DIRS}) -add_definitions(${LLVM_DEFINITIONS} -DNDEBUG) - -add_subdirectory(EvalMaxSAT) -add_subdirectory(src) diff --git a/utils/optimin/EVALMAXSAT_VERSION b/utils/optimin/EVALMAXSAT_VERSION deleted file mode 100644 index d836ff1c..00000000 --- a/utils/optimin/EVALMAXSAT_VERSION +++ /dev/null @@ -1 +0,0 @@ -440bf90edf88f6ab940934129e3c5b3b93764295 diff --git a/utils/optimin/EvalMaxSAT b/utils/optimin/EvalMaxSAT deleted file mode 160000 -Subproject 440bf90edf88f6ab940934129e3c5b3b9376429 diff --git a/utils/optimin/README.md b/utils/optimin/README.md deleted file mode 100644 index 340022b8..00000000 --- a/utils/optimin/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# OptiMin - -OptiMin is a corpus minimizer that uses a -[MaxSAT](https://en.wikipedia.org/wiki/Maximum_satisfiability_problem) solver -to identify a subset of functionally distinct files that exercise different code -paths in a target program. - -Unlike most corpus minimizers, such as `afl-cmin`, OptiMin does not rely on -heuristic and/or greedy algorithms to identify these functionally distinct -files. This means that minimized corpora are generally much smaller than those -produced by other tools. - -## Building - -To build the `optimin` just execute the `build_optimin.sh` script. - -## Running - -Running `optimin` is the same as running `afl-cmin`: - -``` -./optimin -h -OVERVIEW: Optimal corpus minimizer -USAGE: optimin [options] <target program> [target args...] - -OPTIONS: - -Color Options: - - --color - Use colors in output (default=autodetect) - -General options: - - -C - Keep crashing inputs, reject everything else - -O - Use binary-only instrumentation (FRIDA mode) - -Q - Use binary-only instrumentation (QEMU mode) - -U - Use unicorn-based instrumentation (unicorn mode) - -f - Include edge hit counts - -i dir - Input directory - -m megs - Memory limit for child process (default=none) - -o dir - Output directory - -p - Display progress bar - -t msec - Run time limit for child process (default=5000) - -w csv - Weights file - -Generic Options: - - --help - Display available options (--help-hidden for more) - --help-list - Display list of available options (--help-list-hidden for more) - --version - Display the version of this program -``` - -Example: `optimin -i files -o seeds -- ./target @@` - -### Weighted Minimizations - -OptiMin allows for weighted minimizations. For examples, seeds can be weighted -by file size (or execution time), thus preferencing the selection of smaller (or -faster) seeds. - -To perform a weighted minimization, supply a CSV file with the `-w` option. This -CSV file is formatted as follows: - -``` -SEED_1,WEIGHT_1 -SEED_2,WEIGHT_2 -... -SEED_N,WEIGHT_N -``` - -Where `SEED_N` is the file name (**not** path) of a seed in the input directory, -and `WEIGHT_N` is an integer weight. - -## Further Details and Citation - -For more details, see the paper -[Seed Selection for Successful Fuzzing](https://dl.acm.org/doi/10.1145/3460319.3464795). -If you use OptiMin in your research, please cite this paper. - -BibTeX: - -```bibtex -@inproceedings{Herrera:2021:FuzzSeedSelection, - author = {Adrian Herrera and Hendra Gunadi and Shane Magrath and Michael Norrish and Mathias Payer and Antony L. Hosking}, - title = {Seed Selection for Successful Fuzzing}, - booktitle = {30th ACM SIGSOFT International Symposium on Software Testing and Analysis}, - series = {ISSTA}, - year = {2021}, - pages = {230--243}, - numpages = {14}, - location = {Virtual, Denmark}, - publisher = {Association for Computing Machinery}, -} -```
\ No newline at end of file diff --git a/utils/optimin/build_optimin.sh b/utils/optimin/build_optimin.sh deleted file mode 100755 index aee5d0c3..00000000 --- a/utils/optimin/build_optimin.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/sh -# -# american fuzzy lop++ - optimin build script -# ------------------------------------------------ -# -# Originally written by Nathan Voss <njvoss99@gmail.com> -# -# Adapted from code by Andrew Griffiths <agriffiths@google.com> and -# Michal Zalewski -# -# Adapted for AFLplusplus by Dominik Maier <mail@dmnk.co> -# -# Copyright 2017 Battelle Memorial Institute. All rights reserved. -# Copyright 2019-2022 AFLplusplus Project. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# This script builds the OptiMin corpus minimizer. - -EVALMAXSAT_VERSION="$(cat ./EVALMAXSAT_VERSION)" -EVALMAXSAT_REPO="https://github.com/FlorentAvellaneda/EvalMaxSAT" - -echo "=================================================" -echo "OptiMin build script" -echo "=================================================" -echo - -echo "[*] Performing basic sanity checks..." - -PLT=`uname -s` - -if [ ! -f "../../config.h" ]; then - - echo "[-] Error: key files not found - wrong working directory?" - exit 1 - -fi - -LLVM_CONFIG="${LLVM_CONFIG:-llvm-config}" -CMAKECMD=cmake -MAKECMD=make -TARCMD=tar - -if [ "$PLT" = "Darwin" ]; then - CORES=`sysctl -n hw.ncpu` - TARCMD=tar -fi - -if [ "$PLT" = "FreeBSD" ]; then - MAKECMD=gmake - CORES=`sysctl -n hw.ncpu` - TARCMD=gtar -fi - -if [ "$PLT" = "NetBSD" ] || [ "$PLT" = "OpenBSD" ]; then - MAKECMD=gmake - CORES=`sysctl -n hw.ncpu` - TARCMD=gtar -fi - -PREREQ_NOTFOUND= -for i in git $CMAKECMD $MAKECMD $TARCMD; do - - T=`command -v "$i" 2>/dev/null` - - if [ "$T" = "" ]; then - - echo "[-] Error: '$i' not found. Run 'sudo apt-get install $i' or similar." - PREREQ_NOTFOUND=1 - - fi - -done - -if echo "$CC" | grep -qF /afl-; then - - echo "[-] Error: do not use afl-gcc or afl-clang to compile this tool." - PREREQ_NOTFOUND=1 - -fi - -if [ "$PREREQ_NOTFOUND" = "1" ]; then - exit 1 -fi - -echo "[+] All checks passed!" - -echo "[*] Making sure EvalMaxSAT is checked out" - -git status 1>/dev/null 2>/dev/null -if [ $? -eq 0 ]; then - echo "[*] initializing EvalMaxSAT submodule" - git submodule init || exit 1 - git submodule update ./EvalMaxSAT 2>/dev/null # ignore errors -else - echo "[*] cloning EvalMaxSAT" - test -d EvalMaxSAT || { - CNT=1 - while [ '!' -d EvalMaxSAT -a "$CNT" -lt 4 ]; do - echo "Trying to clone EvalMaxSAT (attempt $CNT/3)" - git clone "$EVALMAXSAT_REPO" - CNT=`expr "$CNT" + 1` - done - } -fi - -test -d EvalMaxSAT || { echo "[-] not checked out, please install git or check your internet connection." ; exit 1 ; } -echo "[+] Got EvalMaxSAT." - -cd "EvalMaxSAT" || exit 1 -echo "[*] Checking out $EVALMAXSAT_VERSION" -sh -c 'git stash && git stash drop' 1>/dev/null 2>/dev/null -git checkout "$EVALMAXSAT_VERSION" || exit 1 -cd .. - -echo -echo -echo "[+] EvalMaxSAT successfully prepared!" -echo "[+] Building OptiMin now." -mkdir -p build -cd build || exit 1 -cmake .. -DLLVM_DIR=`$LLVM_CONFIG --cmakedir` || exit 1 -make -j$CORES || exit 1 -cd .. -echo -cp -fv build/src/optimin . || exit 1 -echo "[+] OptiMin successfully built!" diff --git a/utils/optimin/src/CMakeLists.txt b/utils/optimin/src/CMakeLists.txt deleted file mode 100644 index 693f63f2..00000000 --- a/utils/optimin/src/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_executable(optimin OptiMin.cpp) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") - -foreach(LIB MaLib EvalMaxSAT glucose) - target_include_directories(optimin PRIVATE - "${CMAKE_SOURCE_DIR}/EvalMaxSAT/lib/${LIB}/src") - target_link_libraries(optimin ${LIB}) -endforeach(LIB) - -llvm_map_components_to_libnames(LLVM_LIBS support) -target_link_libraries(optimin ${LLVM_LIBS}) - -install(TARGETS optimin RUNTIME DESTINATION bin) diff --git a/utils/optimin/src/OptiMin.cpp b/utils/optimin/src/OptiMin.cpp deleted file mode 100644 index b0082d14..00000000 --- a/utils/optimin/src/OptiMin.cpp +++ /dev/null @@ -1,702 +0,0 @@ -/* - * OptiMin, an optimal fuzzing corpus minimizer. - * - * Author: Adrian Herrera - */ - -#include <cstdint> -#include <cstdlib> -#include <vector> - -#include <llvm/ADT/DenseSet.h> -#include <llvm/ADT/DenseMap.h> -#include <llvm/ADT/SmallVector.h> -#include <llvm/ADT/StringExtras.h> -#include <llvm/ADT/StringMap.h> -#include <llvm/Support/Chrono.h> -#include <llvm/Support/CommandLine.h> -#include <llvm/Support/Error.h> -#include <llvm/Support/FileSystem.h> -#include <llvm/Support/MemoryBuffer.h> -#include <llvm/Support/Path.h> -#include <llvm/Support/Program.h> -#include <llvm/Support/WithColor.h> - -#include "EvalMaxSAT.h" - -using namespace llvm; - -namespace { - -// -------------------------------------------------------------------------- // -// Classes -// -------------------------------------------------------------------------- // - -/// Ensure seed weights default to 1 -class Weight { - - public: - Weight() : Weight(1){}; - Weight(uint32_t V) : Value(V){}; - - operator unsigned() const { - - return Value; - - } - - private: - const unsigned Value; - -}; - -// -------------------------------------------------------------------------- // -// Typedefs -// -------------------------------------------------------------------------- // - -/// AFL tuple (edge) ID -using AFLTupleID = uint32_t; - -/// Pair of tuple ID and hit count -using AFLTuple = std::pair<AFLTupleID, /* Frequency */ unsigned>; - -/// Coverage for a given seed file -using AFLCoverageVector = std::vector<AFLTuple>; - -/// Map seed file paths to its coverage vector -using AFLCoverageMap = StringMap<AFLCoverageVector>; - -/// Map seed file paths to a weight -using WeightsMap = StringMap<Weight>; - -/// A seed identifier in the MaxSAT solver -using SeedID = int; - -/// Associates seed identifiers to seed files -using MaxSATSeeds = - SmallVector<std::pair<SeedID, /* Seed file */ std::string>, 0>; - -/// Set of literal identifiers -using MaxSATSeedSet = DenseSet<SeedID>; - -/// Maps tuple IDs to the literal identifiers that "cover" that tuple -using MaxSATCoverageMap = DenseMap<AFLTupleID, MaxSATSeedSet>; - -// -------------------------------------------------------------------------- // -// Global variables -// -------------------------------------------------------------------------- // - -// This is based on the human class count in `count_class_human[256]` in -// `afl-showmap.c` -static constexpr uint32_t MAX_EDGE_FREQ = 8; - -// The maximum number of failures allowed when parsing a weights file -static constexpr unsigned MAX_WEIGHT_FAILURES = 5; - -static sys::TimePoint<> StartTime, EndTime; -static std::chrono::seconds Duration; - -static std::string ShowmapPath; -static bool TargetArgsHasAtAt = false; -static bool KeepTraces = false; -static bool SkipBinCheck = false; - -static const auto ErrMsg = [] { - - return WithColor(errs(), raw_ostream::RED, /*Bold=*/true) << "[-] "; - -}; - -static const auto WarnMsg = [] { - - return WithColor(errs(), raw_ostream::MAGENTA, /*Bold=*/true) << "[-] "; - -}; - -static const auto SuccMsg = [] { - - return WithColor(outs(), raw_ostream::GREEN, /*Bold=*/true) << "[+] "; - -}; - -static const auto StatMsg = [] { - - return WithColor(outs(), raw_ostream::BLUE, /*Bold=*/true) << "[*] "; - -}; - -static cl::opt<std::string> InputDir("i", cl::desc("Input directory"), - cl::value_desc("dir"), cl::Required); -static cl::opt<std::string> OutputDir("o", cl::desc("Output directory"), - cl::value_desc("dir"), cl::Required); - -static cl::opt<bool> EdgesOnly("f", cl::desc("Include edge hit counts"), - cl::init(true)); -static cl::opt<std::string> WeightsFile("w", cl::desc("Weights file"), - cl::value_desc("csv")); - -static cl::opt<std::string> TargetProg(cl::Positional, - cl::desc("<target program>"), - cl::Required); -static cl::list<std::string> TargetArgs(cl::ConsumeAfter, - cl::desc("[target args...]")); - -static cl::opt<std::string> MemLimit( - "m", cl::desc("Memory limit for child process (default=none)"), - cl::value_desc("megs"), cl::init("none")); -static cl::opt<std::string> Timeout( - "t", cl::desc("Run time limit for child process (default=5000)"), - cl::value_desc("msec"), cl::init("5000")); - -static cl::opt<bool> CrashMode( - "C", cl::desc("Keep crashing inputs, reject everything else")); -static cl::opt<bool> FridaMode( - "O", cl::desc("Use binary-only instrumentation (FRIDA mode)")); -static cl::opt<bool> QemuMode( - "Q", cl::desc("Use binary-only instrumentation (QEMU mode)")); -static cl::opt<bool> UnicornMode( - "U", cl::desc("Use unicorn-based instrumentation (unicorn mode)")); - -} // anonymous namespace - -// -------------------------------------------------------------------------- // -// Helper functions -// -------------------------------------------------------------------------- // - -static void GetWeights(const MemoryBuffer &MB, WeightsMap &Weights) { - - SmallVector<StringRef, 0> Lines; - MB.getBuffer().trim().split(Lines, '\n'); - - unsigned FailureCount = 0; - unsigned Weight = 0; - - for (const auto &Line : Lines) { - - const auto &[Seed, WeightStr] = Line.split(','); - - if (to_integer(WeightStr, Weight, 10)) { - - Weights.try_emplace(Seed, Weight); - - } else { - - if (FailureCount >= MAX_WEIGHT_FAILURES) { - ErrMsg() << "Too many failures. Aborting\n"; - std::exit(1); - } - - WarnMsg() << "Failed to read weight for '" << Seed << "'. Skipping...\n"; - FailureCount++; - - } - - } - -} - -static std::error_code readCov(const StringRef Trace, AFLCoverageVector &Cov) { - - const auto CovOrErr = MemoryBuffer::getFile(Trace); - if (const auto EC = CovOrErr.getError()) return EC; - - SmallVector<StringRef, 0> Lines; - CovOrErr.get()->getBuffer().trim().split(Lines, '\n'); - - AFLTupleID Edge = 0; - unsigned Freq = 0; - - for (const auto &Line : Lines) { - - const auto &[EdgeStr, FreqStr] = Line.split(':'); - - to_integer(EdgeStr, Edge, 10); - to_integer(FreqStr, Freq, 10); - Cov.push_back({Edge, Freq}); - - } - - return std::error_code(); - -} - -static Error runShowmap(AFLCoverageMap &CovMap, const StringRef Input, - bool BinCheck = false) { - - const bool InputIsFile = !sys::fs::is_directory(Input); - Optional<StringRef> Redirects[] = {None, None, None}; - - SmallString<32> TraceDir{OutputDir}; - sys::path::append(TraceDir, ".traces"); - - SmallString<32> Output{TraceDir}; - SmallString<32> StdinFile{TraceDir}; - - // ------------------------------------------------------------------------ // - // Prepare afl-showmap arguments - // - // If the given input is a file, then feed this directly into stdin. - // Otherwise, if it is a directory, specify this on the afl-showmap command - // line. - // ------------------------------------------------------------------------ // - - SmallVector<StringRef, 12> ShowmapArgs{ShowmapPath, "-q", - "-m", MemLimit, - "-t", Timeout}; - - if (InputIsFile) { - - StdinFile = Input; - sys::path::append(Output, - BinCheck ? ".run_test" : sys::path::filename(Input)); - - } else { - - sys::path::append(StdinFile, ".cur_input"); - ShowmapArgs.append({"-i", Input}); - - } - - - if (TargetArgsHasAtAt) { - - ShowmapArgs.append({"-H", StdinFile}); - Redirects[/* stdin */ 0] = "/dev/null"; - - } else if (InputIsFile) { - - Redirects[/* stdin */ 0] = Input; - - } - - if (FridaMode) ShowmapArgs.push_back("-O"); - if (QemuMode) ShowmapArgs.push_back("-Q"); - if (UnicornMode) ShowmapArgs.push_back("-U"); - - ShowmapArgs.append({"-o", Output, "--", TargetProg}); - ShowmapArgs.append(TargetArgs.begin(), TargetArgs.end()); - - // ------------------------------------------------------------------------ // - // Run afl-showmap - // ------------------------------------------------------------------------ // - - const int RC = sys::ExecuteAndWait(ShowmapPath, ShowmapArgs, - /*env=*/None, Redirects); - if (RC && !CrashMode) { - - ErrMsg() << "Exit code " << RC << " != 0 received from afl-showmap\n"; - return createStringError(inconvertibleErrorCode(), "afl-showmap failed"); - - } - - // ------------------------------------------------------------------------ // - // Parse afl-showmap output - // ------------------------------------------------------------------------ // - - AFLCoverageVector Cov; - std::error_code EC; - sys::fs::file_status Status; - - if (InputIsFile) { - - // Read a single output coverage file - if ((EC = readCov(Output, Cov))) { - - sys::fs::remove(Output); - return errorCodeToError(EC); - - } - - CovMap.try_emplace(sys::path::filename(Input), Cov); - if (!KeepTraces) sys::fs::remove(Output); - - } else { - - // Read a directory of output coverage files - for (sys::fs::recursive_directory_iterator Dir(TraceDir, EC), DirEnd; - Dir != DirEnd && !EC; Dir.increment(EC)) { - - if (EC) return errorCodeToError(EC); - - const auto &Path = Dir->path(); - if ((EC = sys::fs::status(Path, Status))) return errorCodeToError(EC); - - switch (Status.type()) { - - case sys::fs::file_type::regular_file: - case sys::fs::file_type::symlink_file: - case sys::fs::file_type::type_unknown: - Cov.clear(); - if ((EC = readCov(Path, Cov))) { - - sys::fs::remove(Path); - return errorCodeToError(EC); - - } - - CovMap.try_emplace(sys::path::filename(Path), Cov); - default: - // Ignore - break; - - } - - } - - if (!KeepTraces) sys::fs::remove_directories(TraceDir); - - } - - return Error::success(); - -} - -static inline void StartTimer() { - - StartTime = std::chrono::system_clock::now(); - -} - -static inline void EndTimer() { - - EndTime = std::chrono::system_clock::now(); - Duration = - std::chrono::duration_cast<std::chrono::seconds>(EndTime - StartTime); - - SuccMsg() << " Completed in " << Duration.count() << " s\n"; - -} - -// -------------------------------------------------------------------------- // -// Main function -// -------------------------------------------------------------------------- // - -int main(int argc, char *argv[]) { - - WeightsMap Weights; - std::error_code EC; - - // ------------------------------------------------------------------------ // - // Parse command-line options and environment variables - // - // Also check the target arguments, as this determines how we run afl-showmap. - // ------------------------------------------------------------------------ // - - cl::ParseCommandLineOptions(argc, argv, "Optimal corpus minimizer"); - - KeepTraces = !!std::getenv("AFL_KEEP_TRACES"); - SkipBinCheck = !!std::getenv("AFL_SKIP_BIN_CHECK"); - const auto AFLPath = std::getenv("AFL_PATH"); - - if (CrashMode) ::setenv("AFL_CMIN_CRASHES_ONLY", "1", /*overwrite=*/true); - - for (const auto &Arg : TargetArgs) - if (Arg == "@@") TargetArgsHasAtAt = true; - - // ------------------------------------------------------------------------ // - // Find afl-showmap - // ------------------------------------------------------------------------ // - - SmallVector<StringRef, 16> EnvPaths; - - if (const char *PathEnv = std::getenv("PATH")) - SplitString(PathEnv, EnvPaths, ":"); - if (AFLPath) EnvPaths.push_back(AFLPath); - - const auto ShowmapOrErr = sys::findProgramByName("afl-showmap", EnvPaths); - if (ShowmapOrErr.getError()) { - - ErrMsg() << "Failed to find afl-showmap. Check your PATH\n"; - return 1; - - } - - ShowmapPath = *ShowmapOrErr; - - // ------------------------------------------------------------------------ // - // Parse weights - // - // Weights are stored in CSV file mapping a seed file name to an integer - // greater than zero. - // ------------------------------------------------------------------------ // - - if (WeightsFile != "") { - - StatMsg() << "Reading weights from '" << WeightsFile << "'...\n"; - StartTimer(); - - const auto WeightsOrErr = MemoryBuffer::getFile(WeightsFile); - if ((EC = WeightsOrErr.getError())) { - - ErrMsg() << "Failed to read weights from '" << WeightsFile - << "': " << EC.message() << '\n'; - return 1; - - } - - GetWeights(*WeightsOrErr.get(), Weights); - - EndTimer(); - - } - - // ------------------------------------------------------------------------ // - // Traverse input directory - // - // Find the seed files inside this directory (and subdirectories). - // ------------------------------------------------------------------------ // - - StatMsg() << "Locating seeds in '" << InputDir << "'...\n"; - StartTimer(); - - bool IsDirResult; - if ((EC = sys::fs::is_directory(InputDir, IsDirResult))) { - - ErrMsg() << "Invalid input directory '" << InputDir << "': " << EC.message() - << '\n'; - return 1; - - } - - sys::fs::file_status Status; - StringMap<std::string> SeedFiles; - - for (sys::fs::recursive_directory_iterator Dir(InputDir, EC), DirEnd; - Dir != DirEnd && !EC; Dir.increment(EC)) { - - if (EC) { - - ErrMsg() << "Failed to traverse input directory '" << InputDir - << "': " << EC.message() << '\n'; - return 1; - - } - - const auto &Path = Dir->path(); - if ((EC = sys::fs::status(Path, Status))) { - - ErrMsg() << "Failed to access '" << Path << "': " << EC.message() << '\n'; - return 1; - - } - - switch (Status.type()) { - - case sys::fs::file_type::regular_file: - case sys::fs::file_type::symlink_file: - case sys::fs::file_type::type_unknown: - SeedFiles.try_emplace(sys::path::filename(Path), - sys::path::parent_path(Path)); - default: - /* Ignore */ - break; - - } - - } - - EndTimer(); - - if (SeedFiles.empty()) { - - ErrMsg() << "Failed to find any seed files in '" << InputDir << "'\n"; - return 1; - - } - - // ------------------------------------------------------------------------ // - // Setup output directory - // ------------------------------------------------------------------------ // - - SmallString<32> TraceDir{OutputDir}; - sys::path::append(TraceDir, ".traces"); - - if ((EC = sys::fs::remove_directories(TraceDir))) { - - ErrMsg() << "Failed to remove existing trace directory in '" << OutputDir - << "': " << EC.message() << '\n'; - return 1; - - } - - if ((EC = sys::fs::create_directories(TraceDir))) { - - ErrMsg() << "Failed to create output directory '" << OutputDir - << "': " << EC.message() << '\n'; - return 1; - - } - - // ------------------------------------------------------------------------ // - // Test the target binary - // ------------------------------------------------------------------------ // - - AFLCoverageMap CovMap; - - if (!SkipBinCheck) { - - const auto It = SeedFiles.begin(); - SmallString<32> TestSeed{It->second}; - sys::path::append(TestSeed, It->first()); - - StatMsg() << "Testing the target binary with '" << TestSeed << "`...\n"; - StartTimer(); - - if (auto Err = runShowmap(CovMap, TestSeed, /*BinCheck=*/true)) { - - ErrMsg() << "No instrumentation output detected \n"; - return 1; - - } - - EndTimer(); - SuccMsg() << "OK, " << CovMap.begin()->second.size() - << " tuples recorded\n"; - - } - - // ------------------------------------------------------------------------ // - // Generate seed coverage - // - // Iterate over the corpus directory, which should contain seed files. Execute - // these seeds in the target program to generate coverage information, and - // then store this coverage information in the appropriate data structures. - // ------------------------------------------------------------------------ // - - StatMsg() << "Running afl-showmap on " << SeedFiles.size() << " seeds...\n"; - StartTimer(); - - MaxSATSeeds SeedVars; - MaxSATCoverageMap SeedCoverage; - EvalMaxSAT Solver(/*nbMinimizeThread=*/0); - - CovMap.clear(); - if (auto Err = runShowmap(CovMap, InputDir)) { - - ErrMsg() << "Failed to generate coverage: " << Err << '\n'; - return 1; - - } - - for (const auto &SeedCov : CovMap) { - - // Create a variable to represent the seed - const SeedID Var = Solver.newVar(); - SeedVars.emplace_back(Var, SeedCov.first()); - - // Record the set of seeds that cover a particular edge - for (auto &[Edge, Freq] : SeedCov.second) { - - if (EdgesOnly) { - - // Ignore edge frequency - SeedCoverage[Edge].insert(Var); - - } else { - - // Executing edge `E` `N` times means that it was executed `N - 1` times - for (unsigned I = 0; I < Freq; ++I) - SeedCoverage[MAX_EDGE_FREQ * Edge + I].insert(Var); - - } - - } - - } - - EndTimer(); - - // ------------------------------------------------------------------------ // - // Set the hard and soft constraints in the solver - // ------------------------------------------------------------------------ // - - StatMsg() << "Generating constraints...\n"; - StartTimer(); - - size_t SeedCount = 0; - - // Ensure that at least one seed is selected that covers a particular edge - // (hard constraint) - std::vector<SeedID> Clauses; - for (const auto &[_, Seeds] : SeedCoverage) { - - if (Seeds.empty()) continue; - - Clauses.clear(); - for (const auto &Seed : Seeds) - Clauses.push_back(Seed); - - Solver.addClause(Clauses); - - } - - // Select the minimum number of seeds that cover a particular set of edges - // (soft constraint) - for (const auto &[Var, Seed] : SeedVars) - Solver.addWeightedClause({-Var}, Weights[sys::path::filename(Seed)]); - - EndTimer(); - - // ------------------------------------------------------------------------ // - // Generate a solution - // ------------------------------------------------------------------------ // - - StatMsg() << "Solving...\n"; - StartTimer(); - - const bool Solved = Solver.solve(); - - EndTimer(); - - // ------------------------------------------------------------------------ // - // Save the solution - // - // This will copy the selected seeds to the given output directory. - // ------------------------------------------------------------------------ // - - SmallVector<StringRef, 64> Solution; - SmallString<32> InputSeed, OutputSeed; - - if (Solved) { - - for (const auto &[Var, Seed] : SeedVars) - if (Solver.getValue(Var) > 0) Solution.push_back(Seed); - - } else { - - ErrMsg() << "Failed to find an optimal solution for '" << InputDir << "'\n"; - return 1; - - } - - StatMsg() << "Copying " << Solution.size() << " seeds to '" << OutputDir - << "'...\n"; - StartTimer(); - - SeedCount = 0; - - for (const auto &Seed : Solution) { - - InputSeed = SeedFiles[Seed]; - sys::path::append(InputSeed, Seed); - - OutputSeed = OutputDir; - sys::path::append(OutputSeed, Seed); - - if ((EC = sys::fs::copy_file(InputSeed, OutputSeed))) { - - ErrMsg() << "Failed to copy '" << Seed << "' to '" << OutputDir - << "': " << EC.message() << '\n'; - return 1; - - } - - } - - EndTimer(); - SuccMsg() << "Done!\n"; - - return 0; - -} - |