Merge pull request #666 from AFLplusplus/dev

push to stable

12 files changed, 104 insertions, 55 deletions

diff --git a/Dockerfile b/Dockerfile
index abb76209..dec952af 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -20,7 +20,7 @@ RUN apt-get update && \
     python3 python3-dev python3-setuptools python-is-python3 \
     libtool libtool-bin \
     libglib2.0-dev \
-    wget vim jupp nano bash-completion \
+    wget vim jupp nano bash-completion less \
     apt-utils apt-transport-https ca-certificates gnupg dialog \
     libpixman-1-dev \
     gnuplot-nox \
diff --git a/GNUmakefile b/GNUmakefile
index db2ad572..7b05a1d5 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -414,7 +414,7 @@ afl-as: src/afl-as.c include/afl-as.h $(COMM_HDR) | test_x86
 	@ln -sf afl-as as
 
 src/afl-performance.o : $(COMM_HDR) src/afl-performance.c include/hash.h
-	$(CC) -Iinclude $(SPECIAL_PERFORMANCE) -O3 -fno-unroll-loops -c src/afl-performance.c -o src/afl-performance.o
+	$(CC) $(CFLAGS) -Iinclude $(SPECIAL_PERFORMANCE) -O3 -fno-unroll-loops -c src/afl-performance.c -o src/afl-performance.o
 
 src/afl-common.o : $(COMM_HDR) src/afl-common.c include/common.h
 	$(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-common.c -o src/afl-common.o
@@ -583,7 +583,7 @@ distrib: all
 	-$(MAKE) -f GNUmakefile.gcc_plugin
 	$(MAKE) -C utils/libdislocator
 	$(MAKE) -C utils/libtokencap
-	$(MAKE) -C utils/aflpp_driver
+	-$(MAKE) -C utils/aflpp_driver
 	$(MAKE) -C utils/afl_network_proxy
 	$(MAKE) -C utils/socket_fuzzing
 	$(MAKE) -C utils/argv_fuzzing
@@ -606,7 +606,7 @@ source-only: all
 	-$(MAKE) -f GNUmakefile.gcc_plugin
 	$(MAKE) -C utils/libdislocator
 	$(MAKE) -C utils/libtokencap
-	$(MAKE) -C utils/aflpp_driver
+	-$(MAKE) -C utils/aflpp_driver
 
 %.8:	%
 	@echo .TH $* 8 $(BUILD_DATE) "afl++" > $@
diff --git a/docs/Changelog.md b/docs/Changelog.md
index a8b8f0af..71846535 100644
--- a/docs/Changelog.md
+++ b/docs/Changelog.md
@@ -10,6 +10,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
 
 
 ### Version ++3.01a (release)
+  - Mac OS ARM64 support
   - afl-fuzz
     - fix crash for very, very fast targets+systems (thanks to mhlakhani
       for reporting)
diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h
index e2fb0344..0bf0aebe 100644
--- a/include/afl-fuzz.h
+++ b/include/afl-fuzz.h
@@ -174,6 +174,10 @@ struct queue_entry {
   u8 *trace_mini;                       /* Trace bytes, if kept             */
   u32 tc_ref;                           /* Trace bytes ref count            */
 
+#ifdef INTROSPECTION
+  u32 bitsmap_size;
+#endif
+
   double perf_score,                    /* performance score                */
       weight;
 
@@ -586,7 +590,8 @@ typedef struct afl_state {
 
   u32 rand_cnt;                         /* Random number counter            */
 
-  u64 rand_seed[3];
+/*  unsigned long rand_seed[3]; would also work */
+  AFL_RAND_RETURN rand_seed[3];
   s64 init_seed;
 
   u64 total_cal_us,                     /* Total calibration time (us)      */
@@ -734,6 +739,7 @@ typedef struct afl_state {
   char  mutation[8072];
   char  m_tmp[4096];
   FILE *introspection_file;
+  u32   bitsmap_size;
 #endif
 
 } afl_state_t;
diff --git a/src/afl-cc.c b/src/afl-cc.c
index 00e9cfce..e6a6718e 100644
--- a/src/afl-cc.c
+++ b/src/afl-cc.c
@@ -879,7 +879,7 @@ static void edit_params(u32 argc, char **argv, char **envp) {
 
 #ifndef __ANDROID__
 
-  if (compiler_mode != GCC) {
+  if (compiler_mode != GCC && compiler_mode != CLANG) {
 
     switch (bit_mode) {
 
@@ -1364,17 +1364,17 @@ int main(int argc, char **argv, char **envp) {
 
     if (clang_mode) {
 
-      instrument_mode = CLANG;
+      instrument_mode = INSTRUMENT_CLANG;
 
     } else {
 
-      instrument_mode = GCC;
+      instrument_mode = INSTRUMENT_GCC;
 
     }
 
   }
 
-  if (compiler_mode == CLANG) { instrument_mode = CLANG; }
+  if (compiler_mode == CLANG) { instrument_mode = INSTRUMENT_CLANG; }
 
   if (argc < 2 || strncmp(argv[1], "-h", 2) == 0) {
 
diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c
index ec937f29..dbffa4f9 100644
--- a/src/afl-fuzz-init.c
+++ b/src/afl-fuzz-init.c
@@ -1667,20 +1667,21 @@ static void handle_existing_out_dir(afl_state_t *afl) {
 
   if (afl->in_place_resume && rmdir(fn)) {
 
-    time_t     cur_t = time(0);
-    struct tm *t = localtime(&cur_t);
+    time_t    cur_t = time(0);
+    struct tm t;
+    localtime_r(&cur_t, &t);
 
 #ifndef SIMPLE_FILES
 
-    u8 *nfn = alloc_printf("%s.%04d-%02d-%02d-%02d:%02d:%02d", fn,
-                           t->tm_year + 1900, t->tm_mon + 1, t->tm_mday,
-                           t->tm_hour, t->tm_min, t->tm_sec);
+    u8 *nfn =
+        alloc_printf("%s.%04d-%02d-%02d-%02d:%02d:%02d", fn, t.tm_year + 1900,
+                     t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec);
 
 #else
 
-    u8 *nfn = alloc_printf("%s_%04d%02d%02d%02d%02d%02d", fn, t->tm_year + 1900,
-                           t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min,
-                           t->tm_sec);
+    u8 *nfn =
+        alloc_printf("%s_%04d%02d%02d%02d%02d%02d", fn, t.tm_year + 1900,
+                     t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec);
 
 #endif                                                    /* ^!SIMPLE_FILES */
 
@@ -1698,20 +1699,21 @@ static void handle_existing_out_dir(afl_state_t *afl) {
 
   if (afl->in_place_resume && rmdir(fn)) {
 
-    time_t     cur_t = time(0);
-    struct tm *t = localtime(&cur_t);
+    time_t    cur_t = time(0);
+    struct tm t;
+    localtime_r(&cur_t, &t);
 
 #ifndef SIMPLE_FILES
 
-    u8 *nfn = alloc_printf("%s.%04d-%02d-%02d-%02d:%02d:%02d", fn,
-                           t->tm_year + 1900, t->tm_mon + 1, t->tm_mday,
-                           t->tm_hour, t->tm_min, t->tm_sec);
+    u8 *nfn =
+        alloc_printf("%s.%04d-%02d-%02d-%02d:%02d:%02d", fn, t.tm_year + 1900,
+                     t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec);
 
 #else
 
-    u8 *nfn = alloc_printf("%s_%04d%02d%02d%02d%02d%02d", fn, t->tm_year + 1900,
-                           t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min,
-                           t->tm_sec);
+    u8 *nfn =
+        alloc_printf("%s_%04d%02d%02d%02d%02d%02d", fn, t.tm_year + 1900,
+                     t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec);
 
 #endif                                                    /* ^!SIMPLE_FILES */
 
diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 54afa17c..9a0d199e 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -45,25 +45,19 @@ inline u32 select_next_queue_entry(afl_state_t *afl) {
 double compute_weight(afl_state_t *afl, struct queue_entry *q,
                       double avg_exec_us, double avg_bitmap_size) {
 
-  u32 hits;
+  double weight = 1.0;
 
   if (likely(afl->schedule >= FAST && afl->schedule <= RARE)) {
 
-    hits = afl->n_fuzz[q->n_fuzz_entry];
-    if (hits == 0) { hits = 1; }
-
-  } else {
-
-    hits = 1;
+    u32 hits = afl->n_fuzz[q->n_fuzz_entry];
+    if (likely(hits)) { weight *= log10(hits) + 1; }
 
   }
 
-  double weight = 1.0;
   weight *= avg_exec_us / q->exec_us;
-  weight *= log(q->bitmap_size) / avg_bitmap_size;
-  weight /= log10(hits) + 1;
+  weight *= (log(q->bitmap_size) / avg_bitmap_size);
 
-  if (q->favored) weight *= 5;
+  if (unlikely(q->favored)) weight *= 5;
 
   return weight;
 
@@ -97,34 +91,43 @@ void create_alias_table(afl_state_t *afl) {
 
     double avg_exec_us = 0.0;
     double avg_bitmap_size = 0.0;
+    u32    active = 0;
+
     for (i = 0; i < n; i++) {
 
       struct queue_entry *q = afl->queue_buf[i];
-      avg_exec_us += q->exec_us;
-      avg_bitmap_size += log(q->bitmap_size);
+
+      // disabled entries might have timings and bitmap values
+      if (likely(!q->disabled)) {
+
+        avg_exec_us += q->exec_us;
+        avg_bitmap_size += log(q->bitmap_size);
+        ++active;
+
+      }
 
     }
 
-    avg_exec_us /= afl->queued_paths;
-    avg_bitmap_size /= afl->queued_paths;
+    avg_exec_us /= active;
+    avg_bitmap_size /= active;
 
     for (i = 0; i < n; i++) {
 
       struct queue_entry *q = afl->queue_buf[i];
 
-      if (!q->disabled) {
+      if (likely(!q->disabled)) {
 
         q->weight = compute_weight(afl, q, avg_exec_us, avg_bitmap_size);
         q->perf_score = calculate_score(afl, q);
+        sum += q->weight;
 
       }
 
-      sum += q->weight;
-
     }
 
     for (i = 0; i < n; i++) {
 
+      // weight is always 0 for disabled entries
       P[i] = (afl->queue_buf[i]->weight * n) / sum;
 
     }
@@ -143,8 +146,8 @@ void create_alias_table(afl_state_t *afl) {
 
     for (i = 0; i < n; i++) {
 
-      struct queue_entry *q = afl->queue_buf[i];
-      P[i] = (q->perf_score * n) / sum;
+      // perf_score is always 0 for disabled entries
+      P[i] = (afl->queue_buf[i]->perf_score * n) / sum;
 
     }
 
@@ -190,11 +193,39 @@ void create_alias_table(afl_state_t *afl) {
   while (nS)
     afl->alias_probability[S[--nS]] = 1;
 
+#ifdef INTROSPECTION
+  u8 fn[PATH_MAX];
+  snprintf(fn, PATH_MAX, "%s/introspection_corpus.txt", afl->out_dir);
+  FILE *f = fopen(fn, "a");
+  if (f) {
+
+    for (i = 0; i < n; i++) {
+
+      struct queue_entry *q = afl->queue_buf[i];
+      fprintf(
+          f,
+          "entry=%u name=%s favored=%s variable=%s disabled=%s len=%u "
+          "exec_us=%u "
+          "bitmap_size=%u bitsmap_size=%u tops=%u weight=%f perf_score=%f\n",
+          i, q->fname, q->favored ? "true" : "false",
+          q->var_behavior ? "true" : "false", q->disabled ? "true" : "false",
+          q->len, (u32)q->exec_us, q->bitmap_size, q->bitsmap_size, q->tc_ref,
+          q->weight, q->perf_score);
+
+    }
+
+    fprintf(f, "\n");
+    fclose(f);
+
+  }
+
+#endif
+
   /*
-  fprintf(stderr, "  entry  alias  probability  perf_score   filename\n");
-  for (u32 i = 0; i < n; ++i)
-    fprintf(stderr, "  %5u  %5u  %11u  %0.9f  %s\n", i, afl->alias_table[i],
-            afl->alias_probability[i], afl->queue_buf[i]->perf_score,
+  fprintf(stderr, "  entry  alias  probability  perf_score   weight
+  filename\n"); for (u32 i = 0; i < n; ++i) fprintf(stderr, "  %5u  %5u  %11u
+  %0.9f  %0.9f  %s\n", i, afl->alias_table[i], afl->alias_probability[i],
+  afl->queue_buf[i]->perf_score, afl->queue_buf[i]->weight,
             afl->queue_buf[i]->fname);
   */
 
@@ -398,6 +429,10 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
   q->trace_mini = NULL;
   q->testcase_buf = NULL;
 
+#ifdef INTROSPECTION
+  q->bitsmap_size = afl->bitsmap_size;
+#endif
+
   if (q->depth > afl->max_depth) { afl->max_depth = q->depth; }
 
   if (afl->queue_top) {
diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c
index 9a9ac33f..37d66aef 100644
--- a/src/afl-fuzz-redqueen.c
+++ b/src/afl-fuzz-redqueen.c
@@ -456,7 +456,8 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
   }
 
-  if (SHAPE_BYTES(h->shape) >= 1 && *status != 1) {
+  /* avoid CodeQL warning on unsigned overflow */
+  if (/* SHAPE_BYTES(h->shape) >= 1 && */ *status != 1) {
 
     if (its_len >= 1 && *buf_8 == (u8)pattern && *o_buf_8 == (u8)o_pattern) {
 
diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c
index d53ba546..339fb9c3 100644
--- a/src/afl-fuzz-run.c
+++ b/src/afl-fuzz-run.c
@@ -380,6 +380,10 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem,
 
     }
 
+#ifdef INTROSPECTION
+    if (unlikely(!q->bitsmap_size)) q->bitsmap_size = afl->bitsmap_size;
+#endif
+
     classify_counts(&afl->fsrv);
     cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST);
     if (q->exec_cksum != cksum) {
diff --git a/test/test-basic.sh b/test/test-basic.sh
index 2ddf14af..79f90ea0 100755
--- a/test/test-basic.sh
+++ b/test/test-basic.sh
@@ -71,7 +71,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc
     test -z "$SKIP" && {
       $ECHO "$GREY[*] running afl-fuzz for ${AFL_GCC}, this will take approx 10 seconds"
       {
-        ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -- ./test-instr.plain >>errors 2>&1
+        ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -D -- ./test-instr.plain >>errors 2>&1
       } >>errors 2>&1
       test -n "$( ls out/default/queue/id:000002* 2>/dev/null )" && {
         $ECHO "$GREEN[+] afl-fuzz is working correctly with ${AFL_GCC}"
@@ -191,7 +191,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc
     test -z "$SKIP" && {
       $ECHO "$GREY[*] running afl-fuzz for ${AFL_GCC}, this will take approx 10 seconds"
       {
-        ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -- ./test-instr.plain >>errors 2>&1
+        ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -D -- ./test-instr.plain >>errors 2>&1
       } >>errors 2>&1
       test -n "$( ls out/default/queue/id:000002* 2>/dev/null )" && {
         $ECHO "$GREEN[+] afl-fuzz is working correctly with ${AFL_GCC}"
diff --git a/test/test-gcc-plugin.sh b/test/test-gcc-plugin.sh
index 2b09e753..9fe63ea3 100755
--- a/test/test-gcc-plugin.sh
+++ b/test/test-gcc-plugin.sh
@@ -67,7 +67,7 @@ test -e ../afl-gcc-fast -a -e ../afl-compiler-rt.o && {
       echo 0 > in/in
       $ECHO "$GREY[*] running afl-fuzz for gcc_plugin, this will take approx 10 seconds"
       {
-        ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -- ./test-instr.plain.gccpi >>errors 2>&1
+        ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -D -- ./test-instr.plain.gccpi >>errors 2>&1
       } >>errors 2>&1
       test -n "$( ls out/default/queue/id:000002* 2>/dev/null )" && {
         $ECHO "$GREEN[+] afl-fuzz is working correctly with gcc_plugin"
diff --git a/test/test-llvm.sh b/test/test-llvm.sh
index 09ade0c3..e5005d72 100755
--- a/test/test-llvm.sh
+++ b/test/test-llvm.sh
@@ -71,7 +71,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && {
     test -z "$SKIP" && {
       $ECHO "$GREY[*] running afl-fuzz for llvm_mode, this will take approx 10 seconds"
       {
-        ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -- ./test-instr.plain >>errors 2>&1
+        ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -D -- ./test-instr.plain >>errors 2>&1
       } >>errors 2>&1
       test -n "$( ls out/default/queue/id:000002* 2>/dev/null )" && {
         $ECHO "$GREEN[+] afl-fuzz is working correctly with llvm_mode"
@@ -164,7 +164,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && {
     echo ZZZZ > in/in
     $ECHO "$GREY[*] running afl-fuzz with floating point splitting, this will take max. 30 seconds"
     {
-      AFL_BENCH_UNTIL_CRASH=1 AFL_NO_UI=1 ../afl-fuzz -s 1 -V30 -m ${MEM_LIMIT} -i in -o out -- ./test-floatingpoint >>errors 2>&1
+      AFL_BENCH_UNTIL_CRASH=1 AFL_NO_UI=1 ../afl-fuzz -s 1 -V30 -m ${MEM_LIMIT} -i in -o out -D -- ./test-floatingpoint >>errors 2>&1
     } >>errors 2>&1
     test -n "$( ls out/default/crashes/id:* 2>/dev/null )" && {
       $ECHO "$GREEN[+] llvm_mode laf-intel floatingpoint splitting feature works correctly"