Refactor to use an alternative method for shared memory.

If USEMMAP is defined, the shared memory segment is created/attached etc.
now by shm_open() and mmap().
This API is hopefully more often available (at least for iOS).

In order to reduce code duplication I have added new files
sharedmem.[ch] which now encapsulate the shared memory method.

This is based on the work of Proteas to support iOS fuzzing (thanks).
https://github.com/Proteas/afl-ios/commit/866af8ad1cb230d5d753b546380a4af1e55d6946

Currently this is in an experimental status yet. Please report
whether this variant works on 32 and 64 bit and on the supported platforms.

This branch enables USEMMAP and has been tested on Linux.
There is no auto detection for the mmap API yet.

1 files changed, 46 insertions, 0 deletions

diff --git a/afl-as.h b/afl-as.h
index ebd57109..90944614 100644
--- a/afl-as.h
+++ b/afl-as.h
@@ -220,6 +220,29 @@ static const u8* main_payload_32 =
   "  testl %eax, %eax\n"
   "  je    __afl_setup_abort\n"
   "\n"
+#ifdef USEMMAP
+  "  pushl $384        /* shm_open mode 0600 */\n"
+  "  pushl $2          /* flags O_RDWR   */\n"
+  "  pushl %eax        /* SHM file path  */\n"
+  "  call  shm_open\n"
+  "  addl  $12, %esp\n"
+  "\n"
+  "  cmpl $-1, %eax\n"
+  "  je   __afl_setup_abort\n"
+  "\n"
+  "  pushl $0          /* mmap off       */\n"
+  "  pushl %eax        /* shm fd         */\n"
+  "  pushl $1          /* mmap flags     */\n"
+  "  pushl $3          /* mmap prot      */\n"
+  "  pushl $"STRINGIFY(MAP_SIZE)"          /* mmap len       */\n"
+  "  pushl $0          /* mmap addr      */\n"
+  "  call  mmap\n"
+  "  addl  $12, %esp\n"
+  "\n"
+  "  cmpl $-1, %eax\n"
+  "  je   __afl_setup_abort\n"
+  "\n"
+#else
   "  pushl %eax\n"
   "  call  atoi\n"
   "  addl  $4, %esp\n"
@@ -233,6 +256,7 @@ static const u8* main_payload_32 =
   "  cmpl $-1, %eax\n"
   "  je   __afl_setup_abort\n"
   "\n"
+#endif
   "  /* Store the address of the SHM region. */\n"
   "\n"
   "  movl %eax, __afl_area_ptr\n"
@@ -501,6 +525,27 @@ static const u8* main_payload_64 =
   "  testq %rax, %rax\n"
   "  je    __afl_setup_abort\n"
   "\n"
+#ifdef USEMMAP
+  "  movl $384, %edx   /* shm_open mode 0600 */\n"
+  "  movl $2,   %esi   /* flags O_RDWR   */\n"
+  "  movq %rax, %rdi   /* SHM file path  */\n"
+  CALL_L64("shm_open")
+  "\n"
+  "  cmpq $-1, %rax\n"
+  "  je   __afl_setup_abort\n"
+  "\n"
+  "  movl    $0, %r9d\n"
+  "  movl    %eax, %r8d\n"
+  "  movl    $1, %ecx\n"
+  "  movl    $3, %edx\n"
+  "  movl    $"STRINGIFY(MAP_SIZE)", %esi\n"
+  "  movl    $0, %edi\n"
+  CALL_L64("mmap")
+  "\n"
+  "  cmpq $-1, %rax\n"
+  "  je   __afl_setup_abort\n"
+  "\n"
+#else
   "  movq  %rax, %rdi\n"
   CALL_L64("atoi")
   "\n"
@@ -512,6 +557,7 @@ static const u8* main_payload_64 =
   "  cmpq $-1, %rax\n"
   "  je   __afl_setup_abort\n"
   "\n"
+#endif
   "  /* Store the address of the SHM region. */\n"
   "\n"
   "  movq %rax, %rdx\n"