Merge branch 'dev' into text_inputs

5 files changed, 220 insertions, 17 deletions

diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c
index e95ae95f..65ad0c9f 100644
--- a/src/afl-fuzz-init.c
+++ b/src/afl-fuzz-init.c
@@ -438,6 +438,159 @@ static void shuffle_ptrs(afl_state_t *afl, void **ptrs, u32 cnt) {
 
 }
 
+/* Read all testcases from foreign input directories, then queue them for
+   testing. Called at startup and at sync intervals.
+   Does not descend into subdirectories! */
+
+void read_foreign_testcases(afl_state_t *afl, int first) {
+
+  if (!afl->foreign_sync_cnt) return;
+
+  struct dirent **nl;
+  s32             nl_cnt;
+  u32             i, iter;
+
+  u8 val_buf[2][STRINGIFY_VAL_SIZE_MAX];
+
+  for (iter = 0; iter < afl->foreign_sync_cnt; iter++) {
+
+    if (afl->foreign_syncs[iter].dir != NULL &&
+        afl->foreign_syncs[iter].dir[0] != 0) {
+
+      if (first) ACTF("Scanning '%s'...", afl->foreign_syncs[iter].dir);
+      time_t ctime_max = 0;
+
+      /* We use scandir() + alphasort() rather than readdir() because otherwise,
+         the ordering of test cases would vary somewhat randomly and would be
+         difficult to control. */
+
+      nl_cnt = scandir(afl->foreign_syncs[iter].dir, &nl, NULL, NULL);
+
+      if (nl_cnt < 0) {
+
+        if (first) {
+
+          WARNF("Unable to open directory '%s'", afl->foreign_syncs[iter].dir);
+          sleep(1);
+
+        }
+
+        continue;
+
+      }
+
+      if (nl_cnt == 0) {
+
+        if (first)
+          WARNF("directory %s is currently empty",
+                afl->foreign_syncs[iter].dir);
+        continue;
+
+      }
+
+      /* Show stats */
+
+      snprintf(afl->stage_name_buf, STAGE_BUF_SIZE, "foreign sync %u", iter);
+
+      afl->stage_name = afl->stage_name_buf;
+      afl->stage_cur = 0;
+      afl->stage_max = 0;
+
+      for (i = 0; i < nl_cnt; ++i) {
+
+        struct stat st;
+
+        u8 *fn2 =
+            alloc_printf("%s/%s", afl->foreign_syncs[iter].dir, nl[i]->d_name);
+
+        free(nl[i]);                                         /* not tracked */
+
+        if (unlikely(lstat(fn2, &st) || access(fn2, R_OK))) {
+
+          if (first) PFATAL("Unable to access '%s'", fn2);
+          continue;
+
+        }
+
+        /* we detect new files by their ctime */
+        if (likely(st.st_ctime <= afl->foreign_syncs[iter].ctime)) {
+
+          ck_free(fn2);
+          continue;
+
+        }
+
+        /* This also takes care of . and .. */
+
+        if (!S_ISREG(st.st_mode) || !st.st_size || strstr(fn2, "/README.txt")) {
+
+          ck_free(fn2);
+          continue;
+
+        }
+
+        if (st.st_size > MAX_FILE) {
+
+          if (first)
+            WARNF(
+                "Test case '%s' is too big (%s, limit is %s), skipping", fn2,
+                stringify_mem_size(val_buf[0], sizeof(val_buf[0]), st.st_size),
+                stringify_mem_size(val_buf[1], sizeof(val_buf[1]), MAX_FILE));
+          ck_free(fn2);
+          continue;
+
+        }
+
+        // lets do not use add_to_queue(afl, fn2, st.st_size, 0);
+        // as this could add duplicates of the startup input corpus
+
+        int fd = open(fn2, O_RDONLY);
+        if (fd < 0) {
+
+          ck_free(fn2);
+          continue;
+
+        }
+
+        u8  fault;
+        u8 *mem = mmap(0, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+
+        if (mem == MAP_FAILED) {
+
+          ck_free(fn2);
+          continue;
+
+        }
+
+        write_to_testcase(afl, mem, st.st_size);
+        fault = fuzz_run_target(afl, &afl->fsrv, afl->fsrv.exec_tmout);
+        afl->syncing_party = "foreign";
+        afl->queued_imported +=
+            save_if_interesting(afl, mem, st.st_size, fault);
+        afl->syncing_party = 0;
+        munmap(mem, st.st_size);
+        close(fd);
+
+        if (st.st_ctime > ctime_max) ctime_max = st.st_ctime;
+
+      }
+
+      afl->foreign_syncs[iter].ctime = ctime_max;
+      free(nl);                                              /* not tracked */
+
+    }
+
+  }
+
+  if (first) {
+
+    afl->last_path_time = 0;
+    afl->queued_at_start = afl->queued_paths;
+
+  }
+
+}
+
 /* Read all testcases from the input directory, then queue them for testing.
    Called at startup. */
 
@@ -466,7 +619,7 @@ void read_testcases(afl_state_t *afl) {
   ACTF("Scanning '%s'...", afl->in_dir);
 
   /* We use scandir() + alphasort() rather than readdir() because otherwise,
-     the ordering  of test cases would vary somewhat randomly and would be
+     the ordering of test cases would vary somewhat randomly and would be
      difficult to control. */
 
   nl_cnt = scandir(afl->in_dir, &nl, NULL, alphasort);
@@ -527,9 +680,11 @@ void read_testcases(afl_state_t *afl) {
 
     if (st.st_size > MAX_FILE) {
 
-      FATAL("Test case '%s' is too big (%s, limit is %s)", fn2,
+      WARNF("Test case '%s' is too big (%s, limit is %s), skipping", fn2,
             stringify_mem_size(val_buf[0], sizeof(val_buf[0]), st.st_size),
             stringify_mem_size(val_buf[1], sizeof(val_buf[1]), MAX_FILE));
+      ck_free(fn2);
+      continue;
 
     }
 
diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c
index 0fb34ab7..b288cf9f 100644
--- a/src/afl-fuzz-mutators.c
+++ b/src/afl-fuzz-mutators.c
@@ -40,7 +40,7 @@ void setup_custom_mutators(afl_state_t *afl) {
 
   if (fn) {
 
-    if (afl->limit_time_sig)
+    if (afl->limit_time_sig && afl->limit_time_sig != -1)
       FATAL(
           "MOpt and custom mutator are mutually exclusive. We accept pull "
           "requests that integrates MOpt with the optional mutators "
@@ -168,7 +168,8 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) {
 
   /* "afl_custom_deinit", optional for backward compatibility */
   mutator->afl_custom_deinit = dlsym(dh, "afl_custom_deinit");
-  if (!mutator->afl_custom_deinit) FATAL("Symbol 'afl_custom_init' not found.");
+  if (!mutator->afl_custom_deinit)
+    FATAL("Symbol 'afl_custom_deinit' not found.");
 
   /* "afl_custom_post_process", optional */
   mutator->afl_custom_post_process = dlsym(dh, "afl_custom_post_process");
@@ -282,9 +283,23 @@ u8 trim_case_custom(afl_state_t *afl, struct queue_entry *q, u8 *in_buf,
 
     } else if (unlikely(retlen > orig_len)) {
 
-      FATAL(
-          "Trimmed data returned by custom mutator is larger than original "
-          "data");
+      /* Do not exit the fuzzer, even if the trimmed data returned by the custom
+         mutator is larger than the original data. For some use cases, like the
+         grammar mutator, the definition of "size" may have different meanings.
+         For example, the trimming function in a grammar mutator aims at
+         reducing the objects in a grammar structure, but does not guarantee to
+         generate a smaller binary buffer.
+
+         Thus, we allow the custom mutator to generate the trimmed data that is
+         larger than the original data. */
+
+      if (afl->not_on_tty && afl->debug) {
+
+        WARNF(
+            "Trimmed data returned by custom mutator is larger than original "
+            "data");
+
+      }
 
     } else if (unlikely(retlen == 0)) {
 
diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c
index 2a1664e2..6e3be72b 100644
--- a/src/afl-fuzz-run.c
+++ b/src/afl-fuzz-run.c
@@ -612,6 +612,8 @@ void sync_fuzzers(afl_state_t *afl) {
 
   }
 
+  if (afl->foreign_sync_cnt) read_foreign_testcases(afl, 0);
+
 }
 
 /* Trim all new test cases to save cycles when doing deterministic checks. The
diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c
index f53205fb..9b16c226 100644
--- a/src/afl-fuzz-stats.c
+++ b/src/afl-fuzz-stats.c
@@ -39,7 +39,7 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability,
   u8                     fn[PATH_MAX];
   s32                    fd;
   FILE *                 f;
-  uint32_t               t_bytes = count_non_255_bytes(afl, afl->virgin_bits);
+  u32                    t_bytes = count_non_255_bytes(afl, afl->virgin_bits);
 
   snprintf(fn, PATH_MAX, "%s/fuzzer_stats", afl->out_dir);
 
@@ -67,6 +67,17 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability,
 
   }
 
+  if ((unlikely(!afl->last_avg_exec_update ||
+                cur_time - afl->last_avg_exec_update >= 60000))) {
+
+    afl->last_avg_execs_saved =
+        (float)(1000 * (afl->fsrv.total_execs - afl->last_avg_execs)) /
+        (float)(cur_time - afl->last_avg_exec_update);
+    afl->last_avg_execs = afl->fsrv.total_execs;
+    afl->last_avg_exec_update = cur_time;
+
+  }
+
 #ifndef __HAIKU__
   if (getrusage(RUSAGE_CHILDREN, &rus)) { rus.ru_maxrss = 0; }
 #endif
@@ -81,7 +92,7 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability,
       "cycles_wo_finds   : %llu\n"
       "execs_done        : %llu\n"
       "execs_per_sec     : %0.02f\n"
-      //          "real_execs_per_sec: %0.02f\n"  // damn the name is too long
+      "execs_ps_last_min : %0.02f\n"
       "paths_total       : %u\n"
       "paths_favored     : %u\n"
       "paths_found       : %u\n"
@@ -117,6 +128,7 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability,
       afl->fsrv.total_execs,
       afl->fsrv.total_execs /
           ((double)(get_cur_time() - afl->start_time) / 1000),
+      afl->last_avg_execs_saved,
       afl->queued_paths, afl->queued_favored, afl->queued_discovered,
       afl->queued_imported, afl->max_depth, afl->current_entry,
       afl->pending_favored, afl->pending_not_fuzzed, afl->queued_variable,
@@ -126,17 +138,17 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability,
       afl->fsrv.exec_tmout, afl->slowest_exec_ms,
 #ifndef __HAIKU__
   #ifdef __APPLE__
-      (unsigned long int)(rus.ru_maxrss >> 20),
+          (unsigned long int)(rus.ru_maxrss >> 20),
   #else
-      (unsigned long int)(rus.ru_maxrss >> 10),
+          (unsigned long int)(rus.ru_maxrss >> 10),
   #endif
 #else
-      -1UL,
+          -1UL,
 #endif
 #ifdef HAVE_AFFINITY
-      afl->cpu_aff,
+          afl->cpu_aff,
 #else
-      -1,
+          -1,
 #endif
       t_bytes, afl->var_byte_count, afl->expand_havoc, afl->use_banner,
       afl->unicorn_mode ? "unicorn" : "", afl->fsrv.qemu_mode ? "qemu " : "",
diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c
index c014035e..5bedf6e1 100644
--- a/src/afl-fuzz.c
+++ b/src/afl-fuzz.c
@@ -131,10 +131,13 @@ static void usage(afl_state_t *afl, u8 *argv0, int more_help) {
       "executions.\n\n"
 
       "Other stuff:\n"
-      "  -T text       - text banner to show on the screen\n"
       "  -M/-S id      - distributed mode (see docs/parallel_fuzzing.md)\n"
       "                  use -D to force -S secondary to perform deterministic "
       "fuzzing\n"
+      "  -F path       - sync to a foreign fuzzer queue directory (requires "
+      "-M, can\n"
+      "                  be specified up to %u times)\n"
+      "  -T text       - text banner to show on the screen\n"
       "  -I command    - execute this command/script when a new crash is "
       "found\n"
       //"  -B bitmap.txt - mutate a specific test case, use the out/fuzz_bitmap
@@ -142,7 +145,7 @@ static void usage(afl_state_t *afl, u8 *argv0, int more_help) {
       "  -C            - crash exploration mode (the peruvian rabbit thing)\n"
       "  -e ext        - file extension for the fuzz test input file (if "
       "needed)\n\n",
-      argv0, EXEC_TIMEOUT, MEM_LIMIT);
+      argv0, EXEC_TIMEOUT, MEM_LIMIT, FOREIGN_SYNCS_MAX);
 
   if (more_help > 1) {
 
@@ -269,7 +272,7 @@ int main(int argc, char **argv_orig, char **envp) {
   afl->shmem_testcase_mode = 1;  // we always try to perform shmem fuzzing
 
   while ((opt = getopt(argc, argv,
-                       "+c:i:I:o:f:m:t:T:dDnCB:S:M:x:QNUWe:p:s:V:E:L:hRP:")) >
+                       "+c:i:I:o:f:F:m:t:T:dDnCB:S:M:x:QNUWe:p:s:V:E:L:hRP:")) >
          0) {
 
     switch (opt) {
@@ -403,6 +406,19 @@ int main(int argc, char **argv_orig, char **envp) {
         afl->use_splicing = 1;
         break;
 
+      case 'F':                                         /* foreign sync dir */
+
+        if (!afl->is_main_node)
+          FATAL(
+              "Option -F can only be specified after the -M option for the "
+              "main fuzzer of a fuzzing campaign");
+        if (afl->foreign_sync_cnt >= FOREIGN_SYNCS_MAX)
+          FATAL("Maximum %u entried of -F option can be specified",
+                FOREIGN_SYNCS_MAX);
+        afl->foreign_syncs[afl->foreign_sync_cnt].dir = optarg;
+        afl->foreign_sync_cnt++;
+        break;
+
       case 'f':                                              /* target file */
 
         if (afl->fsrv.out_file) { FATAL("Multiple -f options not supported"); }
@@ -1060,6 +1076,8 @@ int main(int argc, char **argv_orig, char **envp) {
   setup_cmdline_file(afl, argv + optind);
 
   read_testcases(afl);
+  // read_foreign_testcases(afl, 1); for the moment dont do this
+
   load_auto(afl);
 
   pivot_inputs(afl);
@@ -1217,6 +1235,7 @@ int main(int argc, char **argv_orig, char **envp) {
 
   }
 
+  // (void)nice(-20);  // does not improve the speed
   // real start time, we reset, so this works correctly with -V
   afl->start_time = get_cur_time();