variable map size fix, error reporting through forkserver, code format

9 files changed, 192 insertions, 65 deletions

diff --git a/docs/Changelog.md b/docs/Changelog.md
index ea669eed..e1ca4a10 100644
--- a/docs/Changelog.md
+++ b/docs/Changelog.md
@@ -10,6 +10,13 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
 
 
 ### Version ++2.64d (develop):
+  - afl-fuzz:
+     - AFL_MAP_SIZE was not working correctly
+  - llvm_mode:
+     - if LLVM 11 is installed the posix shm_open+mmap is used and a fixed
+       address for the shared memory map is used as this increases the
+       fuzzing speed
+     - fixes to LTO mode if instrumented edges > MAP_SIZE
   - qemu_mode:
     - add information on PIE/PIC load addresses for 32 bit
     - better dependency checks
@@ -17,11 +24,8 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
     - better dependency checks
   - unicorn_mode:
     - better submodule handling
-  - llvm_mode:
-     - if LLVM 11 is installed the posix shm_open+mmap is used and a fixed
-       address for the shared memory map is used as this increases the
-       fuzzing speed
   - all:
+    - forkserver communication now also used for error reporting
     - fix 32 bit build options
     - make clean now leaves qemu-3.1.1.tar.xz and the unicornafl directory
       intact if in a git/svn checkout - unless "deepclean" is used
diff --git a/include/config.h b/include/config.h
index f11ac919..dca5a8f0 100644
--- a/include/config.h
+++ b/include/config.h
@@ -402,12 +402,26 @@
 
 /* Extended forkserver option values */
 
+/* Reporting errors */
+#define FS_OPT_ERROR 0xf800008f
+#define FS_OPT_GET_ERROR(x) ((x & 0x00ffff00) >> 8)
+#define FS_OPT_SET_ERROR(x) ((x & 0x0000ffff) << 8)
+#define FS_ERROR_MAP_SIZE 1
+#define FS_ERROR_MAP_ADDR 2
+#define FS_ERROR_SHM_OPEN 4
+#define FS_ERROR_SHMAT 8
+#define FS_ERROR_MMAP 16
+
+/* Reporting options */
 #define FS_OPT_ENABLED 0x8f000001
 #define FS_OPT_MAPSIZE 0x40000000
 #define FS_OPT_SNAPSHOT 0x20000000
 #define FS_OPT_AUTODICT 0x10000000
+// FS_OPT_MAX_MAPSIZE is 8388608 = 0x800000 = 2^23 = 1 << 22
+#define FS_OPT_MAX_MAPSIZE ((0x00fffffe >> 1) + 1)
 #define FS_OPT_GET_MAPSIZE(x) (((x & 0x00fffffe) >> 1) + 1)
-#define FS_OPT_SET_MAPSIZE(x) (x <= 1 || x > 0x1000000 ? 0 : ((x - 1) << 1))
+#define FS_OPT_SET_MAPSIZE(x) \
+  (x <= 1 || x > FS_OPT_MAX_MAPSIZE ? 0 : ((x - 1) << 1))
 
 #endif                                                  /* ! _HAVE_CONFIG_H */
 
diff --git a/llvm_mode/afl-llvm-lto-instrumentation.so.cc b/llvm_mode/afl-llvm-lto-instrumentation.so.cc
index dbd4a26e..eefac629 100644
--- a/llvm_mode/afl-llvm-lto-instrumentation.so.cc
+++ b/llvm_mode/afl-llvm-lto-instrumentation.so.cc
@@ -679,7 +679,8 @@ bool AFLLTOPass::runOnModule(Module &M) {
 
   // save highest location ID to global variable
   // do this after each function to fail faster
-  if (!be_quiet && afl_global_id > MAP_SIZE) {
+  if (!be_quiet && afl_global_id > MAP_SIZE &&
+      afl_global_id > FS_OPT_MAX_MAPSIZE) {
 
     uint32_t pow2map = 1, map = afl_global_id;
     while ((map = map >> 1))
@@ -741,18 +742,13 @@ bool AFLLTOPass::runOnModule(Module &M) {
 
       if (afl_global_id % 8) write_loc = (((afl_global_id + 8) >> 3) << 3);
 
-      if (write_loc <= MAP_SIZE && write_loc <= 0x800000) {
-
-        GlobalVariable *AFLFinalLoc = new GlobalVariable(
-            M, Int32Ty, true, GlobalValue::ExternalLinkage, 0,
-            "__afl_final_loc", 0, GlobalVariable::GeneralDynamicTLSModel, 0,
-            false);
-        ConstantInt *const_loc = ConstantInt::get(Int32Ty, write_loc);
-        StoreInst *  StoreFinalLoc = IRB.CreateStore(const_loc, AFLFinalLoc);
-        StoreFinalLoc->setMetadata(M.getMDKindID("nosanitize"),
-                                   MDNode::get(C, None));
-
-      }
+      GlobalVariable *AFLFinalLoc = new GlobalVariable(
+          M, Int32Ty, true, GlobalValue::ExternalLinkage, 0, "__afl_final_loc",
+          0, GlobalVariable::GeneralDynamicTLSModel, 0, false);
+      ConstantInt *const_loc = ConstantInt::get(Int32Ty, write_loc);
+      StoreInst *  StoreFinalLoc = IRB.CreateStore(const_loc, AFLFinalLoc);
+      StoreFinalLoc->setMetadata(M.getMDKindID("nosanitize"),
+                                 MDNode::get(C, None));
 
     }
 
diff --git a/llvm_mode/afl-llvm-rt.o.c b/llvm_mode/afl-llvm-rt.o.c
index 6da41192..8867ae36 100644
--- a/llvm_mode/afl-llvm-rt.o.c
+++ b/llvm_mode/afl-llvm-rt.o.c
@@ -52,9 +52,9 @@
 
 #define CONST_PRIO 5
 
-//#ifndef MAP_FIXED_NOREPLACE
-//#define MAP_FIXED_NOREPLACE MAP_FIXED
-//#endif
+#ifndef MAP_FIXED_NOREPLACE
+#define MAP_FIXED_NOREPLACE MAP_FIXED
+#endif
 
 #include <sys/mman.h>
 #include <fcntl.h>
@@ -73,6 +73,7 @@ u32        __afl_final_loc;
 u32        __afl_prev_ctx;
 u32        __afl_cmp_counter;
 u32        __afl_dictionary_len;
+u32        __afl_map_size = MAP_SIZE;
 u64        __afl_map_addr;
 #else
 __thread PREV_LOC_T __afl_prev_loc[NGRAM_SIZE_MAX];
@@ -80,6 +81,7 @@ __thread u32        __afl_final_loc;
 __thread u32        __afl_prev_ctx;
 __thread u32        __afl_cmp_counter;
 __thread u32        __afl_dictionary_len;
+__thread u32        __afl_map_size = MAP_SIZE;
 __thread u64        __afl_map_addr;
 #endif
 
@@ -89,20 +91,72 @@ struct cmp_map *__afl_cmp_map;
 
 static u8 is_persistent;
 
+/* Error reporting to forkserver controller */
+
+void send_forkserver_error(int error) {
+
+  u32 status;
+  if (!error || error > 0xffff) return;
+  status = (FS_OPT_ERROR | FS_OPT_SET_ERROR(error));
+  if (write(FORKSRV_FD + 1, (char *)&status, 4) != 4) return;
+
+}
+
 /* SHM setup. */
 
 static void __afl_map_shm(void) {
 
-  u8 *         id_str = getenv(SHM_ENV_VAR);
-  unsigned int map_size = MAP_SIZE;
+  char *id_str = getenv(SHM_ENV_VAR);
+
+  if (__afl_final_loc) {
+
+    __afl_map_size = __afl_final_loc;
+    if (__afl_final_loc > MAP_SIZE) {
+
+      char *ptr;
+      u32   val = 0;
+      if ((ptr = getenv("AFL_MAP_SIZE")) != NULL) val = atoi(ptr);
+      if (val < __afl_final_loc) {
+
+        if (__afl_final_loc > FS_OPT_MAX_MAPSIZE) {
+
+          fprintf(stderr,
+                  "Error: AFL++ tools *require* to set AFL_MAP_SIZE to %u to "
+                  "be able to run this instrumented program!\n",
+                  __afl_final_loc);
+          if (id_str) {
+
+            send_forkserver_error(FS_ERROR_MAP_SIZE);
+            exit(-1);
+
+          }
+
+        } else {
+
+          fprintf(stderr,
+                  "Warning: AFL++ tools will need to set AFL_MAP_SIZE to %u to "
+                  "be able to run this instrumented program!\n",
+                  __afl_final_loc);
+
+        }
+
+      }
+
+    }
 
-  if (__afl_final_loc > 1 && __afl_final_loc < MAP_SIZE)
-    map_size = __afl_final_loc;
+  }
 
   /* If we're running under AFL, attach to the appropriate region, replacing the
      early-stage __afl_area_initial region that is needed to allow some really
      hacky .init code to work correctly in projects such as OpenSSL. */
 
+  if (getenv("AFL_DEBUG"))
+    fprintf(stderr,
+            "DEBUG: id_str %s, __afl_map_addr 0x%x, MAP_SIZE %u, "
+            "__afl_final_loc %u, max_size_forkserver %u/0x%x\n",
+            id_str == NULL ? "<null>" : id_str, __afl_map_addr, MAP_SIZE,
+            __afl_final_loc, FS_OPT_MAX_MAPSIZE, FS_OPT_MAX_MAPSIZE);
+
   if (id_str) {
 
 #ifdef USEMMAP
@@ -115,6 +169,7 @@ static void __afl_map_shm(void) {
     if (shm_fd == -1) {
 
       fprintf(stderr, "shm_open() failed\n");
+      send_forkserver_error(FS_ERROR_SHM_OPEN);
       exit(1);
 
     }
@@ -122,13 +177,14 @@ static void __afl_map_shm(void) {
     /* map the shared memory segment to the address space of the process */
     if (__afl_map_addr) {
 
-      shm_base = mmap((void *)__afl_map_addr, map_size, PROT_READ | PROT_WRITE,
-                      MAP_FIXED | MAP_SHARED, shm_fd, 0);
+      shm_base =
+          mmap((void *)__afl_map_addr, __afl_map_size, PROT_READ | PROT_WRITE,
+               MAP_FIXED_NOREPLACE | MAP_SHARED, shm_fd, 0);
 
     } else {
 
-      shm_base =
-          mmap(0, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, 0);
+      shm_base = mmap(0, __afl_map_size, PROT_READ | PROT_WRITE, MAP_SHARED,
+                      shm_fd, 0);
 
     }
 
@@ -138,6 +194,10 @@ static void __afl_map_shm(void) {
       shm_fd = -1;
 
       fprintf(stderr, "mmap() failed\n");
+      if (__afl_map_addr)
+        send_forkserver_error(FS_ERROR_MAP_ADDR);
+      else
+        send_forkserver_error(FS_ERROR_MMAP);
       exit(2);
 
     }
@@ -152,7 +212,15 @@ static void __afl_map_shm(void) {
 
     /* Whooooops. */
 
-    if (__afl_area_ptr == (void *)-1) _exit(1);
+    if (__afl_area_ptr == (void *)-1) {
+
+      if (__afl_map_addr)
+        send_forkserver_error(FS_ERROR_MAP_ADDR);
+      else
+        send_forkserver_error(FS_ERROR_SHMAT);
+      _exit(1);
+
+    }
 
     /* Write something into the bitmap so that even with low AFL_INST_RATIO,
        our parent doesn't give up on us. */
@@ -162,8 +230,8 @@ static void __afl_map_shm(void) {
   } else if (__afl_map_addr) {
 
     __afl_area_ptr =
-        mmap((void *)__afl_map_addr, map_size, PROT_READ | PROT_WRITE,
-             MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+        mmap((void *)__afl_map_addr, __afl_map_size, PROT_READ | PROT_WRITE,
+             MAP_FIXED_NOREPLACE | MAP_SHARED | MAP_ANONYMOUS, -1, 0);
     if (__afl_area_ptr == MAP_FAILED) {
 
       fprintf(stderr, "can not aquire mmap for address %p\n",
@@ -224,13 +292,9 @@ static void __afl_start_snapshots(void) {
   static u8 tmp[4] = {0, 0, 0, 0};
   s32       child_pid;
   u32       status = 0;
-  u32       map_size = MAP_SIZE;
   u32       already_read_first = 0;
   u32       was_killed;
 
-  if (__afl_final_loc > 1 && __afl_final_loc < MAP_SIZE)
-    map_size = __afl_final_loc;
-
   u8 child_stopped = 0;
 
   void (*old_sigchld_handler)(int) = 0;  // = signal(SIGCHLD, SIG_DFL);
@@ -239,8 +303,8 @@ static void __afl_start_snapshots(void) {
      assume we're not running in forkserver mode and just execute program. */
 
   status |= (FS_OPT_ENABLED | FS_OPT_SNAPSHOT);
-  if (map_size <= 0x800000)
-    status |= (FS_OPT_SET_MAPSIZE(map_size) | FS_OPT_MAPSIZE);
+  if (__afl_map_size <= FS_OPT_MAX_MAPSIZE)
+    status |= (FS_OPT_SET_MAPSIZE(__afl_map_size) | FS_OPT_MAPSIZE);
   if (__afl_dictionary_len > 0 && __afl_dictionary) status |= FS_OPT_AUTODICT;
   memcpy(tmp, &status, 4);
 
@@ -393,19 +457,15 @@ static void __afl_start_forkserver(void) {
   u8  tmp[4] = {0, 0, 0, 0};
   s32 child_pid;
   u32 status = 0;
-  u32 map_size = MAP_SIZE;
   u32 already_read_first = 0;
   u32 was_killed;
 
-  if (__afl_final_loc > 1 && __afl_final_loc < MAP_SIZE)
-    map_size = __afl_final_loc;
-
   u8 child_stopped = 0;
 
   void (*old_sigchld_handler)(int) = 0;  // = signal(SIGCHLD, SIG_DFL);
 
-  if (map_size <= 0x800000)
-    status |= (FS_OPT_SET_MAPSIZE(map_size) | FS_OPT_MAPSIZE);
+  if (__afl_map_size <= FS_OPT_MAX_MAPSIZE)
+    status |= (FS_OPT_SET_MAPSIZE(__afl_map_size) | FS_OPT_MAPSIZE);
   if (__afl_dictionary_len > 0 && __afl_dictionary) status |= FS_OPT_AUTODICT;
   if (status) status |= (FS_OPT_ENABLED);
   memcpy(tmp, &status, 4);
@@ -543,12 +603,8 @@ static void __afl_start_forkserver(void) {
 
 int __afl_persistent_loop(unsigned int max_cnt) {
 
-  static u8    first_pass = 1;
-  static u32   cycle_cnt;
-  unsigned int map_size = MAP_SIZE;
-
-  if (__afl_final_loc > 1 && __afl_final_loc < MAP_SIZE)
-    map_size = __afl_final_loc;
+  static u8  first_pass = 1;
+  static u32 cycle_cnt;
 
   if (first_pass) {
 
@@ -559,7 +615,7 @@ int __afl_persistent_loop(unsigned int max_cnt) {
 
     if (is_persistent) {
 
-      memset(__afl_area_ptr, 0, map_size);
+      memset(__afl_area_ptr, 0, __afl_map_size);
       __afl_area_ptr[0] = 1;
       memset(__afl_prev_loc, 0, NGRAM_SIZE_MAX * sizeof(PREV_LOC_T));
 
diff --git a/src/afl-common.c b/src/afl-common.c
index 8d444876..8ae03113 100644
--- a/src/afl-common.c
+++ b/src/afl-common.c
@@ -928,7 +928,8 @@ u32 get_map_size() {
     map_size = atoi(ptr);
     if (map_size < 8 || map_size > (1 << 29)) {
 
-      FATAL("illegal AFL_MAP_SIZE %u, must be between 2^3 and 2^30", map_size);
+      FATAL("illegal AFL_MAP_SIZE %u, must be between %u and %u", map_size, 8,
+            1 << 29);
 
     }
 
diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c
index 0c795f9c..555b82a4 100644
--- a/src/afl-forkserver.c
+++ b/src/afl-forkserver.c
@@ -55,6 +55,8 @@
 
 list_t fsrv_list = {.element_prealloc_count = 0};
 
+void report_error_and_exit(int error);
+
 static void fsrv_exec_child(afl_forkserver_t *fsrv, char **argv) {
 
   execv(fsrv->target_path, argv);
@@ -67,7 +69,6 @@ void afl_fsrv_init(afl_forkserver_t *fsrv) {
 
   // this structure needs default so we initialize it if this was not done
   // already
-
   fsrv->out_fd = -1;
   fsrv->out_dir_fd = -1;
   fsrv->dev_null_fd = -1;
@@ -83,7 +84,7 @@ void afl_fsrv_init(afl_forkserver_t *fsrv) {
 
   /* exec related stuff */
   fsrv->child_pid = -1;
-  fsrv->map_size = MAP_SIZE;
+  fsrv->map_size = get_map_size();
   fsrv->use_fauxsrv = 0;
   fsrv->last_run_timed_out = 0;
 
@@ -201,6 +202,44 @@ static void afl_fauxsrv_execv(afl_forkserver_t *fsrv, char **argv) {
 
 }
 
+/* Report on the error received via the forkserver controller and exit */
+void report_error_and_exit(int error) {
+
+  switch (error) {
+
+    case FS_ERROR_MAP_SIZE:
+      FATAL(
+          "AFL_MAP_SIZE is not set and fuzzing target reports that the "
+          "required size is very large. Solution: Run the fuzzing target "
+          "stand-alone with the environment variable AFL_DEBUG=1 set and set "
+          "the value for __afl_final_loc in the AFL_MAP_SIZE environment "
+          "variable for afl-fuzz.");
+      break;
+    case FS_ERROR_MAP_ADDR:
+      FATAL(
+          "the fuzzing target reports that hardcoded map address might be the "
+          "reason the mmap of the shared memory failed. Solution: recompile "
+          "the target with either afl-clang-lto and the environment variable "
+          "AFL_LLVM_MAP_DYNAMIC set or recompile with afl-clang-fast.");
+      break;
+    case FS_ERROR_SHM_OPEN:
+      FATAL("the fuzzing target reports that the shm_open() call failed.");
+      break;
+    case FS_ERROR_SHMAT:
+      FATAL("the fuzzing target reports that the shmat() call failed.");
+      break;
+    case FS_ERROR_MMAP:
+      FATAL(
+          "the fuzzing target reports that the mmap() call to the share memory "
+          "failed.");
+      break;
+    default:
+      FATAL("unknown error code %u from fuzzing target!", error);
+
+  }
+
+}
+
 /* Spins up fork server (instrumented mode only). The idea is explained here:
 
    http://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html
@@ -400,6 +439,9 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv,
 
     if (!be_quiet) { OKF("All right - fork server is up."); }
 
+    if ((status & FS_OPT_ERROR) == FS_OPT_ERROR)
+      report_error_and_exit(FS_OPT_GET_ERROR(status));
+
     if ((status & FS_OPT_ENABLED) == FS_OPT_ENABLED) {
 
       if (!be_quiet && getenv("AFL_DEBUG")) {
@@ -434,9 +476,10 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv,
 
           FATAL(
               "Target's coverage map size of %u is larger than the one this "
-              "afl++ is set with (%u) (change MAP_SIZE_POW2 in config.h and "
-              "recompile or set AFL_MAP_SIZE)\n",
-              tmp_map_size, fsrv->map_size);
+              "afl++ is set with (%u). Either set AFL_MAP_SIZE=%u and restart "
+              " afl-fuzz, or change MAP_SIZE_POW2 in config.h and recompile "
+              "afl-fuzz",
+              tmp_map_size, fsrv->map_size, tmp_map_size);
 
         }
 
diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c
index b38c9ec5..9f48182b 100644
--- a/src/afl-fuzz-state.c
+++ b/src/afl-fuzz-state.c
@@ -81,7 +81,15 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) {
   and out_size are NULL/0 by default. */
   memset(afl, 0, sizeof(afl_state_t));
 
-  if (!map_size) { afl->shm.map_size = MAP_SIZE; }
+  if (!map_size) {
+
+    afl->shm.map_size = MAP_SIZE;
+
+  } else {
+
+    afl->shm.map_size = map_size;
+
+  }
 
   afl->w_init = 0.9;
   afl->w_end = 0.3;
diff --git a/src/afl-gcc.c b/src/afl-gcc.c
index ac6fdd62..7eb01c0c 100644
--- a/src/afl-gcc.c
+++ b/src/afl-gcc.c
@@ -157,8 +157,7 @@ static void edit_params(u32 argc, char **argv) {
     } else {
 
       fprintf(stderr, "Name of the binary: %s\n", argv[0]);
-      FATAL(
-        "Name of the binary is not a known name, expected afl-clang(++)");
+      FATAL("Name of the binary is not a known name, expected afl-clang(++)");
 
     }
 
@@ -173,15 +172,22 @@ static void edit_params(u32 argc, char **argv) {
 #ifdef __APPLE__
 
     if (!strcmp(name, "afl-g++")) {
+
       cc_params[0] = getenv("AFL_CXX");
+
     } else if (!strcmp(name, "afl-gcj")) {
+
       cc_params[0] = getenv("AFL_GCJ");
+
     } else if (!strcmp(name, "afl-gcc")) {
+
       cc_params[0] = getenv("AFL_CC");
+
     } else {
+
       fprintf(stderr, "Name of the binary: %s\n", argv[0]);
-      FATAL(
-        "Name of the binary is not a known name, expected afl-gcc/g++/gcj");
+      FATAL("Name of the binary is not a known name, expected afl-gcc/g++/gcj");
+
     }
 
     if (!cc_params[0]) {
@@ -218,8 +224,7 @@ static void edit_params(u32 argc, char **argv) {
     } else {
 
       fprintf(stderr, "Name of the binary: %s\n", argv[0]);
-      FATAL(
-        "Name of the binary is not a known name, expected afl-gcc/g++/gcj");
+      FATAL("Name of the binary is not a known name, expected afl-gcc/g++/gcj");
 
     }
 
diff --git a/test/test-compcov.c b/test/test-compcov.c
index 5278af93..c8dd674e 100644
--- a/test/test-compcov.c
+++ b/test/test-compcov.c
@@ -25,7 +25,7 @@ int main(int argc, char **argv) {
     printf("your string was bugmenot\n");
   else if (strcmp(input, "BUFFEROVERFLOW") == 0) {
 
-    buf = (char*)malloc(16);
+    buf = (char *)malloc(16);
     strcpy(buf, "TEST");
     strcat(buf, input);
     printf("This will only crash with libdislocator: %s\n", buf);