add afl-ld-lto for LTO

5 files changed, 440 insertions, 11 deletions

diff --git a/docs/Changelog.md b/docs/Changelog.md
index 87e02938..eaaeb529 100644
--- a/docs/Changelog.md
+++ b/docs/Changelog.md
@@ -22,6 +22,8 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
       better coverage. The original afl instrumentation can be set via
       AFL_LLVM_INSTRUMENT=AFL. This is automatically done when the WHITELIST
       feature is used.
+    - some targets want a ld variant for LD that is not gcc/clang but ld, added
+      afl-ld-lto to solve this
     - lowered minimum required llvm version to 3.4 (except LLVMInsTrim,
       which needs 3.8.0)
     - WHITELIST feature now supports wildcards (thanks to sirmc)
diff --git a/llvm_mode/GNUmakefile b/llvm_mode/GNUmakefile
index 5efe1e12..0e554a39 100644
--- a/llvm_mode/GNUmakefile
+++ b/llvm_mode/GNUmakefile
@@ -246,7 +246,7 @@ ifeq "$(TEST_MMAP)" "1"
         LDFLAGS += -Wno-deprecated-declarations
 endif
 
-  PROGS      = ../afl-clang-fast ../afl-llvm-pass.so ../afl-llvm-lto-whitelist.so ../afl-llvm-lto-instrumentation.so ../afl-llvm-lto-instrim.so ../libLLVMInsTrim.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so ../cmplog-routines-pass.so ../cmplog-instructions-pass.so
+  PROGS      = ../afl-clang-fast ../afl-llvm-pass.so ../afl-ld-lto ../afl-llvm-lto-whitelist.so ../afl-llvm-lto-instrumentation.so ../afl-llvm-lto-instrim.so ../libLLVMInsTrim.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so ../cmplog-routines-pass.so ../cmplog-instructions-pass.so
 
 # If prerequisites are not given, warn, do not build anything, and exit with code 0
 ifeq "$(LLVMVER)" ""
@@ -304,7 +304,7 @@ afl-common.o: ../src/afl-common.c
 	$(CC) $(CFLAGS) -c $< -o $@ $(LDFLAGS)
 
 ../afl-clang-fast: afl-clang-fast.c afl-common.o | test_deps
-	$(CC) $(CFLAGS) $< afl-common.o -o $@ $(LDFLAGS) -DCFLAGS_OPT=\"$(CFLAGS_OPT)\" -Dxxx
+	$(CC) $(CFLAGS) $< afl-common.o -o $@ $(LDFLAGS) -DCFLAGS_OPT=\"$(CFLAGS_OPT)\"
 	ln -sf afl-clang-fast ../afl-clang-fast++
 ifneq "$(AFL_CLANG_FLTO)" ""
 ifeq "$(LLVM_LTO)" "1"
@@ -330,6 +330,11 @@ ifeq "$(LLVM_LTO)" "1"
 	$(CXX) $(CLANG_CFL) -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) afl-llvm-common.o
 endif
 
+../afl-ld-lto: afl-ld-lto.c
+ifeq "$(LLVM_LTO)" "1"
+	$(CC) $(CFLAGS) $< -o $@
+endif
+
 ../afl-llvm-lto-instrumentation.so: afl-llvm-lto-instrumentation.so.cc afl-llvm-common.o
 ifeq "$(LLVM_LTO)" "1"
 	$(CXX) $(CLANG_CFL) -Wno-writable-strings -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) afl-llvm-common.o
@@ -392,6 +397,7 @@ install: all
 	install -d -m 755 $${DESTDIR}$(BIN_PATH) $${DESTDIR}$(HELPER_PATH) $${DESTDIR}$(DOC_PATH) $${DESTDIR}$(MISC_PATH)
 	if [ -f ../afl-clang-fast -a -f ../libLLVMInsTrim.so -a -f ../afl-llvm-rt.o ]; then set -e; install -m 755 ../afl-clang-fast $${DESTDIR}$(BIN_PATH); ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-fast++; install -m 755 ../libLLVMInsTrim.so ../afl-llvm-pass.so ../afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH); fi
 	if [ -f ../afl-clang-lto ]; then set -e; ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-lto; ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-lto++; install -m 755 ../afl-llvm-lto-instrumentation.so ../afl-llvm-lto-instrim.so ../afl-llvm-rt-lto*.o ../afl-llvm-lto-whitelist.so $${DESTDIR}$(HELPER_PATH); fi
+	if [ -f ../afl-ld-lto ]; then set -e; install -m 755 ../afl-ld-lto $${DESTDIR}$(BIN_PATH); fi
 	if [ -f ../afl-llvm-rt-32.o ]; then set -e; install -m 755 ../afl-llvm-rt-32.o $${DESTDIR}$(HELPER_PATH); fi
 	if [ -f ../afl-llvm-rt-64.o ]; then set -e; install -m 755 ../afl-llvm-rt-64.o $${DESTDIR}$(HELPER_PATH); fi
 	if [ -f ../compare-transform-pass.so ]; then set -e; install -m 755 ../compare-transform-pass.so $${DESTDIR}$(HELPER_PATH); fi
diff --git a/llvm_mode/README.lto.md b/llvm_mode/README.lto.md
index 48c587eb..517cb62a 100644
--- a/llvm_mode/README.lto.md
+++ b/llvm_mode/README.lto.md
@@ -14,9 +14,11 @@ This version requires a current llvm 11 compiled from the github master.
 
 4. AUTODICTIONARY feature! see below
 
-5. If any problems arise be sure to set `AR=llvm-ar RANLIB=llvm-ranlib` also
-   note that if that target uses _init functions or early constructors then
-   also set `AFL_LLVM_MAP_DYNAMIC=1` as your target will crash otherwise
+5. If any problems arise be sure to set `AR=llvm-ar RANLIB=llvm-ranlib`.
+   Some targets might need `LD=afl-clang-lto` and others `LD=afl-ld-lto`.
+
+6. If a target uses _init functions or early constructors then additionally
+   set `AFL_LLVM_MAP_DYNAMIC=1` as your target will crash otherwise!
 
 ## Introduction and problem description
 
@@ -61,7 +63,8 @@ AUTODICTIONARY: 11 strings found
 
 ## Getting llvm 11
 
-### Installing llvm 11
+### Installing llvm 11 from the llvm repository
+
 Installing the llvm snapshot builds is easy and mostly painless:
 
 In the follow line change `NAME` for your Debian or Ubuntu release name
@@ -80,7 +83,7 @@ apt-get install -y clang-11 clang-tools-11 libc++1-11 libc++-11-dev \
     libomp5-11 lld-11 lldb-11 llvm-11 llvm-11-dev llvm-11-runtime llvm-11-tools
 ```
 
-### Building llvm 11
+### Building llvm 11 yourself
 
 Building llvm from github takes quite some long time and is not painless:
 ```
@@ -117,6 +120,9 @@ export AFL_LLVM_INSTRUMENT=CFG
 make
 ```
 
+NOTE: some targets also need to set the linker, try both `afl-clang-lto` and
+`afl-ld-lto` for this for `LD=` for `configure`.
+
 ## AUTODICTIONARY feature
 
 Setting `AFL_LLVM_LTO_AUTODICTIONARY` will generate a dictionary in the
@@ -135,6 +141,51 @@ to be dynamic - the original afl way, which is slower).
 AFL_LLVM_MAP_DYNAMIC can be set so the shared memory address is dynamic (which
 is safer but also slower).
 
+## Solving difficult targets
+
+Some targets are difficult because the configure script does unusual stuff that
+is unexpected for afl. See the next chapter `Potential issues` how to solve
+these.
+
+An example of a hard to solve target is ffmpeg. Here is how to successfully
+instrument it:
+
+1. Get and extract the current ffmpeg and change to it's directory
+
+2. Running configure with --cc=clang fails and various other items will fail
+   when compiling, so we have to trick configure:
+
+```
+./configure --enable-lto --disable-shared
+```
+
+3. Now the configuration is done - and we edit the settings in `./ffbuild/config.mak`
+   (-: the original line, +: what to change it into):
+```
+-CC=gcc
++CC=afl-clang-lto
+-CXX=g++
++CXX=afl-clang-lto++
+-AS=gcc
++AS=llvm-as
+-LD=gcc
++LD=afl-clang-lto++
+-DEPCC=gcc
++DEPCC=afl-clang-lto
+-DEPAS=gcc
++DEPAS=afl-clang-lto++
+-AR=ar
++AR=llvm-ar
+-AR_CMD=ar
++AR_CMD=llvm-ar
+-NM_CMD=nm -g
++NM_CMD=llvm-nm -g
+-RANLIB=ranlib -D
++RANLIB=llvm-ranlib -D
+```
+
+4. Then type make, wait for a long time and you are done :)
+
 ## Potential issues
 
 ### compiling libraries fails
@@ -154,6 +205,16 @@ and on some target you have to to AR=/RANLIB= even for make as the configure scr
 Other targets ignore environment variables and need the parameters set via
 `./configure --cc=... --cxx= --ranlib= ...` etc. (I am looking at you ffmpeg!).
 
+
+If you see this message
+```
+assembler command failed ...
+```
+then try setting `llvm-as` for configure:
+```
+AS=llvm-as  ...
+```
+
 ### compiling programs still fail
 
 afl-clang-lto is still work in progress.
@@ -166,11 +227,12 @@ Hence if building a target with afl-clang-lto fails try to build it with llvm11
 and LTO enabled (`CC=clang-11` `CXX=clang++-11` `CFLAGS=-flto=full` and
 `CXXFLAGS=-flto=full`).
 
-An example that does not build with llvm 11 and LTO is ffmpeg.
-
 If this succeeeds then there is an issue with afl-clang-lto. Please report at
 [https://github.com/AFLplusplus/AFLplusplus/issues/226](https://github.com/AFLplusplus/AFLplusplus/issues/226)
 
+Even some targets where clang-11 fails can be build if the fail is just in
+`./configure`, see `Solving difficult targets` above.
+
 ### Target crashes immediately
 
 If the target is using early constructors (priority values smaller than 6)
diff --git a/llvm_mode/afl-ld-lto.c b/llvm_mode/afl-ld-lto.c
new file mode 100644
index 00000000..1b59bb4a
--- /dev/null
+++ b/llvm_mode/afl-ld-lto.c
@@ -0,0 +1,358 @@
+/*
+  american fuzzy lop++ - wrapper for llvm 11+ lld
+  -----------------------------------------------
+
+  Written by Marc Heuse <mh@mh-sec.de> for afl++
+
+  Maintained by Marc Heuse <mh@mh-sec.de>,
+                Heiko Eißfeldt <heiko.eissfeldt@hexco.de>
+                Andrea Fioraldi <andreafioraldi@gmail.com>
+                Dominik Maier <domenukk@gmail.com>
+
+  Copyright 2019-2020 AFLplusplus Project. All rights reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at:
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  The sole purpose of this wrapper is to preprocess clang LTO files when
+  linking with lld and performing the instrumentation on the whole program.
+
+*/
+
+#define AFL_MAIN
+#define _GNU_SOURCE
+
+#include "config.h"
+#include "types.h"
+#include "debug.h"
+#include "alloc-inl.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <ctype.h>
+#include <fcntl.h>
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/time.h>
+
+#include <dirent.h>
+
+#define MAX_PARAM_COUNT 4096
+
+static u8 **ld_params;              /* Parameters passed to the real 'ld'   */
+
+static u8 *afl_path = AFL_PATH;
+static u8 *real_ld = AFL_REAL_LD;
+
+static u8 be_quiet,                 /* Quiet mode (no stderr output)        */
+    debug,                          /* AFL_DEBUG                            */
+    passthrough,                    /* AFL_LD_PASSTHROUGH - no link+optimize*/
+    just_version;                   /* Just show version?                   */
+
+static u32 ld_param_cnt = 1;        /* Number of params to 'ld'             */
+
+/* Examine and modify parameters to pass to 'ld', 'llvm-link' and 'llmv-ar'.
+   Note that the file name is always the last parameter passed by GCC,
+   so we exploit this property to keep the code "simple". */
+static void edit_params(int argc, char **argv) {
+
+  u32 i, instrim = 0, gold_pos = 0, gold_present = 0, rt_present = 0,
+         rt_lto_present = 0, inst_present = 0;
+  char *ptr;
+
+  ld_params = ck_alloc(4096 * sizeof(u8 *));
+
+  ld_params[0] = (u8 *)real_ld;
+
+  if (!passthrough) {
+
+    for (i = 1; i < argc; i++) {
+
+      if (strstr(argv[i], "/afl-llvm-rt-lto.o") != NULL) rt_lto_present = 1;
+      if (strstr(argv[i], "/afl-llvm-rt.o") != NULL) rt_present = 1;
+      if (strstr(argv[i], "/afl-llvm-lto-instr") != NULL) inst_present = 1;
+
+    }
+
+    for (i = 1; i < argc && !gold_pos; i++) {
+
+      if (strcmp(argv[i], "-plugin") == 0) {
+
+        if (strncmp(argv[i], "-plugin=", strlen("-plugin=")) == 0) {
+
+          if (strcasestr(argv[i], "LLVMgold.so") != NULL)
+            gold_present = gold_pos = i + 1;
+
+        } else if (i < argc && strcasestr(argv[i + 1], "LLVMgold.so") != NULL) {
+
+          gold_present = gold_pos = i + 2;
+
+        }
+
+      }
+
+    }
+
+    if (!gold_pos) {
+
+      for (i = 1; i + 1 < argc && !gold_pos; i++) {
+
+        if (argv[i][0] != '-') {
+
+          if (argv[i - 1][0] == '-') {
+
+            switch (argv[i - 1][1]) {
+
+              case 'b':
+                break;
+              case 'd':
+                break;
+              case 'e':
+                break;
+              case 'F':
+                break;
+              case 'f':
+                break;
+              case 'I':
+                break;
+              case 'l':
+                break;
+              case 'L':
+                break;
+              case 'm':
+                break;
+              case 'o':
+                break;
+              case 'O':
+                break;
+              case 'p':
+                if (index(argv[i - 1], '=') == NULL) gold_pos = i;
+                break;
+              case 'R':
+                break;
+              case 'T':
+                break;
+              case 'u':
+                break;
+              case 'y':
+                break;
+              case 'z':
+                break;
+              case '-': {
+
+                if (strcmp(argv[i - 1], "--oformat") == 0) break;
+                if (strcmp(argv[i - 1], "--output") == 0) break;
+                if (strncmp(argv[i - 1], "--opt-remarks-", 14) == 0) break;
+                gold_pos = i;
+                break;
+
+              }
+
+              default:
+                gold_pos = i;
+
+            }
+
+          } else
+
+            gold_pos = i;
+
+        }
+
+      }
+
+    }
+
+    if (!gold_pos) gold_pos = 1;
+
+  }
+
+  if (getenv("AFL_LLVM_INSTRIM"))
+    instrim = 1;
+  else if ((ptr = getenv("AFL_LLVM_INSTRUMENT")) &&
+           (strcasestr(ptr, "CFG") == 0 || strcasestr(ptr, "INSTRIM") == 0))
+    instrim = 1;
+
+  if (debug)
+    SAYF(cMGN "[D] " cRST
+              "passthrough=%s instrim=%d, gold_pos=%d, gold_present=%s "
+              "inst_present=%s rt_present=%s rt_lto_present=%s\n",
+         passthrough ? "true" : "false", instrim, gold_pos,
+         gold_present ? "true" : "false", inst_present ? "true" : "false",
+         rt_present ? "true" : "false", rt_lto_present ? "true" : "false");
+
+  for (i = 1; i < argc; i++) {
+
+    if (ld_param_cnt >= MAX_PARAM_COUNT)
+      FATAL(
+          "Too many command line parameters because of unpacking .a archives, "
+          "this would need to be done by hand ... sorry! :-(");
+
+    if (strcmp(argv[i], "--afl") == 0) {
+
+      if (!be_quiet) OKF("afl++ test command line flag detected, exiting.");
+      exit(0);
+
+    }
+
+    if (i == gold_pos && !passthrough) {
+
+      ld_params[ld_param_cnt++] = alloc_printf("-L%s/../lib", LLVM_BINDIR);
+
+      if (!gold_present) {
+
+        ld_params[ld_param_cnt++] = "-plugin";
+        ld_params[ld_param_cnt++] =
+            alloc_printf("%s/../lib/LLVMgold.so", LLVM_BINDIR);
+
+      }
+
+      ld_params[ld_param_cnt++] = "--allow-multiple-definition";
+
+      if (!inst_present) {
+
+        if (instrim)
+          ld_params[ld_param_cnt++] =
+              alloc_printf("-mllvm=-load=%s/afl-llvm-lto-instrim.so", afl_path);
+        else
+          ld_params[ld_param_cnt++] = alloc_printf(
+              "-mllvm=-load=%s/afl-llvm-lto-instrumentation.so", afl_path);
+
+      }
+
+      if (!rt_present)
+        ld_params[ld_param_cnt++] = alloc_printf("%s/afl-llvm-rt.o", afl_path);
+      if (!rt_lto_present)
+        ld_params[ld_param_cnt++] =
+            alloc_printf("%s/afl-llvm-rt-lto.o", afl_path);
+
+    }
+
+    ld_params[ld_param_cnt++] = argv[i];
+
+  }
+
+  ld_params[ld_param_cnt] = NULL;
+
+}
+
+/* Main entry point */
+
+int main(int argc, char **argv) {
+
+  s32  pid, i, status;
+  u8 * ptr;
+  char thecwd[PATH_MAX];
+
+  if ((ptr = getenv("AFL_LD_CALLER")) != NULL) {
+
+    FATAL("ld loop detected! Set AFL_REAL_LD!\n");
+
+  }
+
+  if (isatty(2) && !getenv("AFL_QUIET") && !getenv("AFL_DEBUG")) {
+
+    SAYF(cCYA "afl-ld-to" VERSION cRST
+              " by Marc \"vanHauser\" Heuse <mh@mh-sec.de>\n");
+
+  } else
+
+    be_quiet = 1;
+
+  if (getenv("AFL_DEBUG") != NULL) debug = 1;
+  if (getenv("AFL_PATH") != NULL) afl_path = getenv("AFL_PATH");
+  if (getenv("AFL_LD_PASSTHROUGH") != NULL) passthrough = 1;
+  if (getenv("AFL_REAL_LD") != NULL) real_ld = getenv("AFL_REAL_LD");
+
+  if (!afl_path || !*afl_path) afl_path = "/usr/local/lib/afl";
+
+  setenv("AFL_LD_CALLER", "1", 1);
+
+  if (debug) {
+
+    (void)getcwd(thecwd, sizeof(thecwd));
+
+    SAYF(cMGN "[D] " cRST "cd \"%s\";", thecwd);
+    for (i = 0; i < argc; i++)
+      SAYF(" \"%s\"", argv[i]);
+    SAYF("\n");
+
+  }
+
+  if (argc < 2) {
+
+    SAYF(
+        "\n"
+        "This is a helper application for afl-clang-lto. It is a wrapper "
+        "around GNU "
+        "llvm's 'lld',\n"
+        "executed by the toolchain whenever using "
+        "afl-clang-lto/afl-clang-lto++.\n"
+        "You probably don't want to run this program directly but rather pass "
+        "it as LD parameter to configure scripts\n\n"
+
+        "Environment variables:\n"
+        "  AFL_LD_PASSTHROUGH   do not link+optimize == no instrumentation\n"
+        "  AFL_REAL_LD          point to the real llvm 11 lld if necessary\n"
+
+        "\nafl-ld-to was compiled with the fixed real 'ld' of %s and the "
+        "binary path of %s\n\n",
+        real_ld, LLVM_BINDIR);
+
+    exit(1);
+
+  }
+
+  edit_params(argc, argv);  // here most of the magic happens :-)
+
+  if (debug) {
+
+    SAYF(cMGN "[D]" cRST " cd \"%s\";", thecwd);
+    for (i = 0; i < ld_param_cnt; i++)
+      SAYF(" \"%s\"", ld_params[i]);
+    SAYF("\n");
+
+  }
+
+  if (!(pid = fork())) {
+
+    if (strlen(real_ld) > 1) execvp(real_ld, (char **)ld_params);
+    execvp("ld", (char **)ld_params);  // fallback
+    FATAL("Oops, failed to execute 'ld' - check your PATH");
+
+  }
+
+  if (pid < 0) PFATAL("fork() failed");
+
+  if (waitpid(pid, &status, 0) <= 0) PFATAL("waitpid() failed");
+  if (debug) SAYF(cMGN "[D] " cRST "linker result: %d\n", status);
+
+  if (!just_version) {
+
+    if (status == 0) {
+
+      if (!be_quiet) OKF("Linker was successful");
+
+    } else {
+
+      SAYF(cLRD "[-] " cRST
+                "Linker failed, please investigate and send a bug report. Most "
+                "likely an 'ld' option is incompatible with %s.\n",
+           AFL_CLANG_FLTO);
+
+    }
+
+  }
+
+  exit(WEXITSTATUS(status));
+
+}
+
diff --git a/src/afl-common.c b/src/afl-common.c
index f4cba573..2802cda3 100644
--- a/src/afl-common.c
+++ b/src/afl-common.c
@@ -61,8 +61,9 @@ char *afl_environment_variables[] = {
     "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", "AFL_IMPORT_FIRST",
     "AFL_INST_LIBS", "AFL_INST_RATIO", "AFL_KEEP_TRACES", "AFL_KEEP_ASSEMBLY",
     "AFL_LD_HARD_FAIL", "AFL_LD_LIMIT_MB", "AFL_LD_NO_CALLOC_OVER",
-    "AFL_LD_PRELOAD", "AFL_LD_VERBOSE", "AFL_LLVM_CMPLOG", "AFL_LLVM_INSTRIM",
-    "AFL_LLVM_CTX", "AFL_LLVM_INSTRUMENT", "AFL_LLVM_INSTRIM_LOOPHEAD",
+    "AFL_LD_PASSTHROUGH", "AFL_REAL_LD", "AFL_LD_PRELOAD", "AFL_LD_VERBOSE",
+    "AFL_LLVM_CMPLOG", "AFL_LLVM_INSTRIM", "AFL_LLVM_CTX",
+    "AFL_LLVM_INSTRUMENT", "AFL_LLVM_INSTRIM_LOOPHEAD",
     "AFL_LLVM_LTO_AUTODICTIONARY", "AFL_LLVM_AUTODICTIONARY",
     "AFL_LLVM_SKIPSINGLEBLOCK", "AFL_LLVM_INSTRIM_SKIPSINGLEBLOCK",
     "AFL_LLVM_LAF_SPLIT_COMPARES", "AFL_LLVM_LAF_SPLIT_COMPARES_BITW",