Merge pull request #654 from AFLplusplus/dev

fix LTO

12 files changed, 81 insertions, 69 deletions

diff --git a/GNUmakefile b/GNUmakefile
index a1af1fd5..db2ad572 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -307,7 +307,7 @@ all:	test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_bu
 
 .PHONY: llvm
 llvm:
-	-$(MAKE) -f GNUmakefile.llvm
+	-$(MAKE) -j -f GNUmakefile.llvm
 	@test -e afl-cc || { echo "[-] Compiling afl-cc failed. You seem not to have a working compiler." ; exit 1; }
 
 .PHONY: gcc_plugin
@@ -579,7 +579,7 @@ deepclean:	clean
 
 .PHONY: distrib
 distrib: all
-	-$(MAKE) -f GNUmakefile.llvm
+	-$(MAKE) -j -f GNUmakefile.llvm
 	-$(MAKE) -f GNUmakefile.gcc_plugin
 	$(MAKE) -C utils/libdislocator
 	$(MAKE) -C utils/libtokencap
@@ -602,7 +602,7 @@ binary-only: test_shm test_python ready $(PROGS)
 
 .PHONY: source-only
 source-only: all
-	-$(MAKE) -f GNUmakefile.llvm
+	-$(MAKE) -j -f GNUmakefile.llvm
 	-$(MAKE) -f GNUmakefile.gcc_plugin
 	$(MAKE) -C utils/libdislocator
 	$(MAKE) -C utils/libtokencap
diff --git a/README.md b/README.md
index a0e7a7e4..bea673f9 100644
--- a/README.md
+++ b/README.md
@@ -31,9 +31,8 @@ With afl++ 3.0 we introduced changes that break some previous afl and afl++
 behaviours and defaults:
 
   * There are no llvm_mode and gcc_plugin subdirectories anymore and there is
-    only one compiler: afl-cc. All previous compilers now symlink to this one
-    compiler. All instrumentation source code is now in the `instrumentation/`
-    folder.
+    only one compiler: afl-cc. All previous compilers now symlink to this.
+    All instrumentation source code is now in the `instrumentation/` folder.
   * The gcc_plugin was replaced with a new version submitted by AdaCore that
     supports more features. thank you!
   * qemu_mode got upgraded to QEMU 5.1, but to be able to build this a current
@@ -41,8 +40,9 @@ behaviours and defaults:
     qemu_mode also got new options like snapshotting, instrumenting specific
     shared libraries, etc. Additionally QEMU 5.1 supports more CPU targets so
     this is really worth it.
-  * When instrumenting targets, afl-cc will not supersede optimizations. This
-    allows to fuzz targets as same as they are built for debug or release.
+  * When instrumenting targets, afl-cc will not supersede optimizations anymore
+    if any were given. This allows to fuzz targets as same as they are built
+    for debug or release.
   * afl-fuzz:
     * if neither -M or -S is specified, `-S default` is assumed, so more
       fuzzers can easily be added later
@@ -88,7 +88,7 @@ behaviours and defaults:
   | Ngram prev_loc Coverage  |         |     x(6)  |            |                  |              |
   | Context Coverage         |         |     x(6)  |            |                  |              |
   | Auto Dictionary          |         |     x(7)  |            |                  |              |
-  | Snapshot LKM Support     |         |     x     |     x      |        (x)(5)    |              |
+  | Snapshot LKM Support     |         |     x(8)  |     x(8)   |        (x)(5)    |              |
 
   1. default for LLVM >= 9.0, env var for older version due an efficiency bug in llvm <= 8
   2. GCC creates non-performant code, hence it is disabled in gcc_plugin
@@ -97,6 +97,7 @@ behaviours and defaults:
   5. upcoming, development in the branch
   6. not compatible with LTO instrumentation and needs at least LLVM >= 4.1
   7. automatic in LTO mode with LLVM >= 11, an extra pass for all LLVM version that writes to a file to use with afl-fuzz' `-x`
+  8. the snapshot LKM is currently unmaintained due to too many kernel changes coming too fast :-(
 
   Among others, the following features and patches have been integrated:
 
@@ -139,9 +140,6 @@ behaviours and defaults:
 
 ## Help wanted
 
-We were happy to be part of [Google Summer of Code 2020](https://summerofcode.withgoogle.com/organizations/5100744400699392/)
-and we will try to participate again in 2021!
-
 We have several ideas we would like to see in AFL++ to make it even better.
 However, we already work on so many things that we do not have the time for
 all the big ideas.
@@ -206,7 +204,7 @@ These build targets exist:
 afl++ binaries by passing the STATIC=1 argument to make:
 
 ```shell
-make all STATIC=1
+make STATIC=1
 ```
 
 These build options exist:
@@ -283,9 +281,9 @@ anything below 9 is not recommended.
     |
     v
  +--------------------------------+
- | if you want to instrument only | -> use GCC_PLUGIN mode (afl-gcc-fast/afl-g++-fast)
- | parts of the target            |    see [instrumentation/README.gcc_plugin.md](instrumentation/README.gcc_plugin.md) and
- +--------------------------------+    [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md)
+ | gcc 5+ is available            | -> use GCC_PLUGIN mode (afl-gcc-fast/afl-g++-fast)
+ +--------------------------------+    see [instrumentation/README.gcc_plugin.md](instrumentation/README.gcc_plugin.md) and
+                                       [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md)
     |
     | if not, or if you do not have a gcc with plugin support
     |
@@ -298,17 +296,17 @@ Clickable README links for the chosen compiler:
   * [LTO mode - afl-clang-lto](instrumentation/README.lto.md)
   * [LLVM mode - afl-clang-fast](instrumentation/README.llvm.md)
   * [GCC_PLUGIN mode - afl-gcc-fast](instrumentation/README.gcc_plugin.md)
-  * GCC mode (afl-gcc) has no README as it has no own features
+  * GCC/CLANG mode (afl-gcc/afl-clang) have no README as they have no own features
 
 You can select the mode for the afl-cc compiler by:
-  1. passing --afl-MODE command line options to the compiler via CFLAGS/CXXFLAGS/CPPFLAGS
-  2. use a symlink to afl-cc: afl-gcc, afl-g++, afl-clang, afl-clang++,
+  1. use a symlink to afl-cc: afl-gcc, afl-g++, afl-clang, afl-clang++,
      afl-clang-fast, afl-clang-fast++, afl-clang-lto, afl-clang-lto++,
-     afl-gcc-fast, afl-g++-fast
-  3. using the environment variable AFL_CC_COMPILER with MODE
+     afl-gcc-fast, afl-g++-fast (recommended!)
+  2. using the environment variable AFL_CC_COMPILER with MODE
+  3. passing --afl-MODE command line options to the compiler via CFLAGS/CXXFLAGS/CPPFLAGS
 
 MODE can be one of: LTO (afl-clang-lto*), LLVM (afl-clang-fast*), GCC_PLUGIN
-(afl-g*-fast) or GCC (afl-gcc/afl-g++).
+(afl-g*-fast) or GCC (afl-gcc/afl-g++) or CLANG(afl-clang/afl-clang++).
 
 Because no afl specific command-line options are accepted (beside the
 --afl-MODE command), the compile-time tools make fairly broad use of environment
@@ -338,14 +336,14 @@ The following options are available when you instrument with LTO mode (afl-clang
    You can read more about this in [instrumentation/README.cmplog.md](instrumentation/README.cmplog.md)
 
 If you use LTO, LLVM or GCC_PLUGIN mode (afl-clang-fast/afl-clang-lto/afl-gcc-fast)
- you have the option to selectively only instrument parts of the target that you
+you have the option to selectively only instrument parts of the target that you
 are interested in:
 
  * To instrument only those parts of the target that you are interested in
    create a file with all the filenames of the source code that should be
    instrumented.
-   For afl-clang-lto and afl-gcc-fast - or afl-clang-fast if either the clang
-   version is below 7 or the CLASSIC instrumentation is used - just put one
+   For afl-clang-lto and afl-gcc-fast - or afl-clang-fast if a mode other than
+   DEFAULT/PCGUARD is used or you have llvm > 10.0.0 - just put one
    filename or function per line (no directory information necessary for
    filenames9, and either set `export AFL_LLVM_ALLOWLIST=allowlist.txt` **or**
    `export AFL_LLVM_DENYLIST=denylist.txt` - depending on if you want per
@@ -353,10 +351,6 @@ are interested in:
    unless requested (ALLOWLIST).
    **NOTE:** During optimization functions might be inlined and then would not match!
    See [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md)
-   For afl-clang-fast > 6.0 or if PCGUARD instrumentation is used then use the
-   llvm sancov allow-list feature: [http://clang.llvm.org/docs/SanitizerCoverage.html](http://clang.llvm.org/docs/SanitizerCoverage.html)
-   The llvm sancov format works with the allowlist/denylist feature of afl++
-   however afl++'s format is more flexible.
 
 There are many more options and modes available however these are most of the
 time less effective. See:
@@ -696,7 +690,7 @@ Note that there are also a lot of tools out there that help fuzzing with afl++
 (some might be deprecated or unsupported):
 
 Minimization of test cases:
- * [afl-pytmin](https://github.com/ilsani/afl-pytmin) - a wrapper for afl-tmin that tries to speed up the process of the minimization of test case by using many CPU cores.
+ * [afl-pytmin](https://github.com/ilsani/afl-pytmin) - a wrapper for afl-tmin that tries to speed up the process of minimization of a single test case by using many CPU cores.
  * [afl-ddmin-mod](https://github.com/MarkusTeufelberger/afl-ddmin-mod) - a variation of afl-tmin based on the ddmin algorithm. 
  * [halfempty](https://github.com/googleprojectzero/halfempty) -  is a fast utility for minimizing test cases by Tavis Ormandy based on parallelization. 
 
@@ -751,7 +745,7 @@ the speed compared to qemu_mode (but slower than persistent mode).
 ### Unicorn
 
 For non-Linux binaries you can use afl++'s unicorn mode which can emulate
-anything you want - for the price of speed and the user writing scripts.
+anything you want - for the price of speed and user written scripts.
 See [unicorn_mode](unicorn_mode/README.md).
 
 It can be easily built by:
@@ -763,16 +757,16 @@ cd unicorn_mode
 ### Shared libraries
 
 If the goal is to fuzz a dynamic library then there are two options available.
-For both you need to write a small hardness that loads and calls the library.
+For both you need to write a small harness that loads and calls the library.
 Faster is the frida solution: [utils/afl_frida/README.md](utils/afl_frida/README.md)
 
 Another, less precise and slower option is using ptrace with debugger interrupt
-instrumentation: [utils/afl_untracer/README.md](utils/afl_untracer/README.md)
+instrumentation: [utils/afl_untracer/README.md](utils/afl_untracer/README.md).
 
 ### More
 
 A more comprehensive description of these and other options can be found in
-[docs/binaryonly_fuzzing.md](docs/binaryonly_fuzzing.md)
+[docs/binaryonly_fuzzing.md](docs/binaryonly_fuzzing.md).
 
 ## Challenges of guided fuzzing
 
diff --git a/docs/Changelog.md b/docs/Changelog.md
index cf9bfbe1..c1cd2d5a 100644
--- a/docs/Changelog.md
+++ b/docs/Changelog.md
@@ -22,6 +22,9 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
     - added AFL_LLVM_INSTRUMENT option NATIVE for native clang pc-guard
       support (less performant than our own), GCC for old afl-gcc and
       CLANG for old afl-clang
+  - warn on any _AFL and __AFL env var
+  - LLVM mode is now compiled with -j4, unicorn with all cores. qemu was
+    already building with all cores, the gcc plugin needs only one.
   - added dummy Makefile to instrumentation/
 
 
diff --git a/include/alloc-inl.h b/include/alloc-inl.h
index 8a91d196..c914da5f 100644
--- a/include/alloc-inl.h
+++ b/include/alloc-inl.h
@@ -363,7 +363,8 @@ static inline void *DFL_ck_realloc(void *orig, u32 size) {
 
   if (orig) {
 
-    memcpy((char *)ret + ALLOC_OFF_HEAD, (char *)orig + ALLOC_OFF_HEAD, MIN(size, old_size));
+    memcpy((char *)ret + ALLOC_OFF_HEAD, (char *)orig + ALLOC_OFF_HEAD,
+           MIN(size, old_size));
     memset((char *)orig + ALLOC_OFF_HEAD, 0xFF, old_size);
 
     ALLOC_C1((char *)orig + ALLOC_OFF_HEAD) = ALLOC_MAGIC_F;
diff --git a/src/afl-analyze.c b/src/afl-analyze.c
index a6825ef6..6dac415b 100644
--- a/src/afl-analyze.c
+++ b/src/afl-analyze.c
@@ -103,11 +103,11 @@ static u32 map_size = MAP_SIZE;
 /* Classify tuple counts. This is a slow & naive version, but good enough here.
  */
 
-#define TIMES4(x) x,x,x,x
-#define TIMES8(x) TIMES4(x),TIMES4(x)
-#define TIMES16(x) TIMES8(x),TIMES8(x)
-#define TIMES32(x) TIMES16(x),TIMES16(x)
-#define TIMES64(x) TIMES32(x),TIMES32(x)
+#define TIMES4(x) x, x, x, x
+#define TIMES8(x) TIMES4(x), TIMES4(x)
+#define TIMES16(x) TIMES8(x), TIMES8(x)
+#define TIMES32(x) TIMES16(x), TIMES16(x)
+#define TIMES64(x) TIMES32(x), TIMES32(x)
 static u8 count_class_lookup[256] = {
 
     [0] = 0,
@@ -121,6 +121,7 @@ static u8 count_class_lookup[256] = {
     [128] = TIMES64(128)
 
 };
+
 #undef TIMES64
 #undef TIMES32
 #undef TIMES16
diff --git a/src/afl-cc.c b/src/afl-cc.c
index 66f4860f..00e9cfce 100644
--- a/src/afl-cc.c
+++ b/src/afl-cc.c
@@ -1030,9 +1030,9 @@ int main(int argc, char **argv, char **envp) {
 
     compiler_mode = GCC;
 
-  } else if (strncmp(callname, "afl-clang", 9) == 0 &&
+  } else if (strcmp(callname, "afl-clang") == 0 ||
 
-             strstr(callname, "fast") == NULL) {
+             strcmp(callname, "afl-clang++") == 0) {
 
     compiler_mode = CLANG;
 
@@ -1076,13 +1076,13 @@ int main(int argc, char **argv, char **envp) {
 
   }
 
-  if (strncmp(callname, "afl-clang", 9) == 0 &&
-      strstr(callname, "fast") == NULL) {
+  if (strcmp(callname, "afl-clang") == 0 ||
+      strcmp(callname, "afl-clang++") == 0) {
 
     clang_mode = 1;
     compiler_mode = CLANG;
 
-    if (strncmp(callname, "afl-clang++", 11) == 0) { plusplus_mode = 1; }
+    if (strcmp(callname, "afl-clang++") == 0) { plusplus_mode = 1; }
 
   }
 
diff --git a/src/afl-common.c b/src/afl-common.c
index 6dc8abe0..7914f83a 100644
--- a/src/afl-common.c
+++ b/src/afl-common.c
@@ -432,7 +432,9 @@ void check_environment_vars(char **envp) {
   char *env, *val;
   while ((env = envp[index++]) != NULL) {
 
-    if (strncmp(env, "ALF_", 4) == 0) {
+    if (strncmp(env, "ALF_", 4) == 0 || strncmp(env, "_ALF", 4) == 0 ||
+        strncmp(env, "__ALF", 5) == 0 || strncmp(env, "_AFL", 4) == 0 ||
+        strncmp(env, "__AFL", 5) == 0) {
 
       WARNF("Potentially mistyped AFL environment variable: %s", env);
       issue_detected = 1;
diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c
index 1cb9b15f..62a8211c 100644
--- a/src/afl-fuzz-bitmap.c
+++ b/src/afl-fuzz-bitmap.c
@@ -26,7 +26,7 @@
 #include "afl-fuzz.h"
 #include <limits.h>
 #if !defined NAME_MAX
-#define NAME_MAX _XOPEN_NAME_MAX
+  #define NAME_MAX _XOPEN_NAME_MAX
 #endif
 
 /* Write bitmap to file. The bitmap is useful mostly for the secret
@@ -143,12 +143,14 @@ u32 count_non_255_bytes(afl_state_t *afl, u8 *mem) {
    and replacing it with 0x80 or 0x01 depending on whether the tuple
    is hit or not. Called on every new crash or timeout, should be
    reasonably fast. */
-#define TIMES4(x) x,x,x,x
-#define TIMES8(x) TIMES4(x),TIMES4(x)
-#define TIMES16(x) TIMES8(x),TIMES8(x)
-#define TIMES32(x) TIMES16(x),TIMES16(x)
-#define TIMES64(x) TIMES32(x),TIMES32(x)
-#define TIMES255(x) TIMES64(x),TIMES64(x),TIMES64(x),TIMES32(x),TIMES16(x),TIMES8(x),TIMES4(x),x,x,x
+#define TIMES4(x) x, x, x, x
+#define TIMES8(x) TIMES4(x), TIMES4(x)
+#define TIMES16(x) TIMES8(x), TIMES8(x)
+#define TIMES32(x) TIMES16(x), TIMES16(x)
+#define TIMES64(x) TIMES32(x), TIMES32(x)
+#define TIMES255(x)                                                      \
+  TIMES64(x), TIMES64(x), TIMES64(x), TIMES32(x), TIMES16(x), TIMES8(x), \
+      TIMES4(x), x, x, x
 const u8 simplify_lookup[256] = {
 
     [0] = 1, [1] = TIMES255(128)
@@ -172,6 +174,7 @@ const u8 count_class_lookup8[256] = {
     [128] = TIMES64(128)
 
 };
+
 #undef TIMES255
 #undef TIMES64
 #undef TIMES32
diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c
index 32cca579..d53ba546 100644
--- a/src/afl-fuzz-run.c
+++ b/src/afl-fuzz-run.c
@@ -29,7 +29,7 @@
 #include <signal.h>
 #include <limits.h>
 #if !defined NAME_MAX
-#define NAME_MAX _XOPEN_NAME_MAX
+  #define NAME_MAX _XOPEN_NAME_MAX
 #endif
 
 #include "cmplog.h"
diff --git a/src/afl-showmap.c b/src/afl-showmap.c
index b891632a..355b2dc3 100644
--- a/src/afl-showmap.c
+++ b/src/afl-showmap.c
@@ -98,17 +98,23 @@ static sharedmem_t *     shm_fuzz;
 /* Classify tuple counts. Instead of mapping to individual bits, as in
    afl-fuzz.c, we map to more user-friendly numbers between 1 and 8. */
 
-#define TIMES4(x) x,x,x,x
-#define TIMES8(x) TIMES4(x),TIMES4(x)
-#define TIMES16(x) TIMES8(x),TIMES8(x)
-#define TIMES32(x) TIMES16(x),TIMES16(x)
-#define TIMES64(x) TIMES32(x),TIMES32(x)
-#define TIMES96(x) TIMES64(x),TIMES32(x)
-#define TIMES128(x) TIMES64(x),TIMES64(x)
+#define TIMES4(x) x, x, x, x
+#define TIMES8(x) TIMES4(x), TIMES4(x)
+#define TIMES16(x) TIMES8(x), TIMES8(x)
+#define TIMES32(x) TIMES16(x), TIMES16(x)
+#define TIMES64(x) TIMES32(x), TIMES32(x)
+#define TIMES96(x) TIMES64(x), TIMES32(x)
+#define TIMES128(x) TIMES64(x), TIMES64(x)
 static const u8 count_class_human[256] = {
 
-    [0] = 0,          [1] = 1,        [2] = 2,         [3] = 3,
-    [4] = TIMES4(4),  [8] = TIMES8(5),[16] = TIMES16(6),[32] = TIMES96(7),
+    [0] = 0,
+    [1] = 1,
+    [2] = 2,
+    [3] = 3,
+    [4] = TIMES4(4),
+    [8] = TIMES8(5),
+    [16] = TIMES16(6),
+    [32] = TIMES96(7),
     [128] = TIMES128(8)
 
 };
@@ -126,6 +132,7 @@ static const u8 count_class_binary[256] = {
     [128] = TIMES64(128)
 
 };
+
 #undef TIMES128
 #undef TIMES96
 #undef TIMES64
diff --git a/src/afl-tmin.c b/src/afl-tmin.c
index 6cb0d458..ed928c7c 100644
--- a/src/afl-tmin.c
+++ b/src/afl-tmin.c
@@ -98,11 +98,11 @@ static sharedmem_t *     shm_fuzz;
 /* Classify tuple counts. This is a slow & naive version, but good enough here.
  */
 
-#define TIMES4(x) x,x,x,x
-#define TIMES8(x) TIMES4(x),TIMES4(x)
-#define TIMES16(x) TIMES8(x),TIMES8(x)
-#define TIMES32(x) TIMES16(x),TIMES16(x)
-#define TIMES64(x) TIMES32(x),TIMES32(x)
+#define TIMES4(x) x, x, x, x
+#define TIMES8(x) TIMES4(x), TIMES4(x)
+#define TIMES16(x) TIMES8(x), TIMES8(x)
+#define TIMES32(x) TIMES16(x), TIMES16(x)
+#define TIMES64(x) TIMES32(x), TIMES32(x)
 static const u8 count_class_lookup[256] = {
 
     [0] = 0,
@@ -116,6 +116,7 @@ static const u8 count_class_lookup[256] = {
     [128] = TIMES64(128)
 
 };
+
 #undef TIMES64
 #undef TIMES32
 #undef TIMES16
diff --git a/unicorn_mode/UNICORNAFL_VERSION b/unicorn_mode/UNICORNAFL_VERSION
index 99025a06..f5537ed8 100644
--- a/unicorn_mode/UNICORNAFL_VERSION
+++ b/unicorn_mode/UNICORNAFL_VERSION
@@ -1 +1 @@
-8cca4801
+768e6bb2