From 4b3ad5f037ee9a36aa057bf55a69acca1f573922 Mon Sep 17 00:00:00 2001
From: vanhauser-thc <vh@thc.org>
Date: Sat, 5 Sep 2020 12:32:10 +0200
Subject: add cull queue, -i subdir traversal

---
 src/afl-fuzz-queue.c | 7 -------
 1 file changed, 7 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index c6d8225f..db91813b 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -239,13 +239,6 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
 
   afl->cycles_wo_finds = 0;
 
-  if (!(afl->queued_paths % 100)) {
-
-    afl->q_prev100->next_100 = q;
-    afl->q_prev100 = q;
-
-  }
-
   struct queue_entry **queue_buf = afl_realloc(
       AFL_BUF_PARAM(queue), afl->queued_paths * sizeof(struct queue_entry *));
   if (unlikely(!queue_buf)) { PFATAL("alloc"); }
-- 
cgit v1.2.3


From e30b2c6af6e369844c92c00a20ebdd53473a747c Mon Sep 17 00:00:00 2001
From: vanhauser-thc <vh@thc.org>
Date: Sat, 5 Sep 2020 13:18:28 +0200
Subject: final changes for pre-3.0

---
 src/afl-fuzz-queue.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index db91813b..af52aa45 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -138,8 +138,7 @@ static u8 check_if_text(struct queue_entry *q) {
     }
 
     // non-overlong 2-byte
-    if (len - offset > 1 &&
-        ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) &&
+    if (len - offset > 1 && ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) &&
          (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF))) {
 
       offset += 2;
@@ -230,7 +229,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
 
   } else {
 
-    afl->q_prev100 = afl->queue = afl->queue_top = q;
+    afl->queue = afl->queue_top = q;
 
   }
 
@@ -274,15 +273,15 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
 
 void destroy_queue(afl_state_t *afl) {
 
-  struct queue_entry *q = afl->queue, *n;
+  struct queue_entry *q;
+  u32                 i;
 
-  while (q) {
+  for (i = 0; i < afl->queued_paths; i++) {
 
-    n = q->next;
+    q = afl->queue_buf[i];
     ck_free(q->fname);
     ck_free(q->trace_mini);
     ck_free(q);
-    q = n;
 
   }
 
-- 
cgit v1.2.3


From 2f90f2faba92c0ef5e081ff74b54fb07eb1faaa9 Mon Sep 17 00:00:00 2001
From: vanhauser-thc <vh@thc.org>
Date: Sat, 5 Sep 2020 13:19:19 +0200
Subject: code-format

---
 src/afl-fuzz-queue.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index af52aa45..8c7bfc55 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -138,7 +138,8 @@ static u8 check_if_text(struct queue_entry *q) {
     }
 
     // non-overlong 2-byte
-    if (len - offset > 1 && ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) &&
+    if (len - offset > 1 &&
+        ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) &&
          (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF))) {
 
       offset += 2;
-- 
cgit v1.2.3


From 6404abd7d609350ffd6c6f221cbf56e60b2ef030 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Mon, 7 Sep 2020 17:30:28 +0200
Subject: bugfix for fixed seeds

---
 src/afl-fuzz-queue.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 8c7bfc55..336b7f4f 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -502,7 +502,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
   // Longer execution time means longer work on the input, the deeper in
   // coverage, the better the fuzzing, right? -mh
 
-  if (afl->schedule >= RARE && likely(!afl->fixed_seed)) {
+  if (likely(afl->schedule < RARE) && likely(!afl->fixed_seed)) {
 
     if (q->exec_us * 0.1 > avg_exec_us) {
 
-- 
cgit v1.2.3


From 7ff9800804f6f4f88b70d8aaf882b66e0ce40c8f Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Sat, 19 Sep 2020 15:13:15 +0200
Subject: fix exploit, mmopt and rare schedule

---
 src/afl-fuzz-queue.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 336b7f4f..ddd08f1c 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -710,7 +710,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
   }
 
-  if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) {
+  if (unlikely(afl->schedule >= EXPLOIT && afl->schedule <= QUAD)) {
 
     if (factor > MAX_FACTOR) { factor = MAX_FACTOR; }
     perf_score *= factor / POWER_BETA;
-- 
cgit v1.2.3


From 888d63748a3c6aafd974cb9d96cdb8d3916e82bb Mon Sep 17 00:00:00 2001
From: Vitalii Akolzin <vva1994@ispras.ru>
Date: Thu, 24 Sep 2020 18:25:32 +0300
Subject: Fix potential endless loop in custom_mutator_stage

Co-authored-by: Ivan Gulakov <gulakov@ispras.ru>
---
 src/afl-fuzz-queue.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index ddd08f1c..14aa34fc 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -234,6 +234,8 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
 
   }
 
+  if (q->len > 1) afl->ready_for_splicing_count++;
+
   ++afl->queued_paths;
   ++afl->pending_not_fuzzed;
 
-- 
cgit v1.2.3


From 6b3b1775b6b274bc62f9c79f686fc79fa110d0a8 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Fri, 25 Sep 2020 12:03:24 +0200
Subject: improving on splice candidate check patch

---
 src/afl-fuzz-queue.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 14aa34fc..53c3e984 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -234,7 +234,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
 
   }
 
-  if (q->len > 1) afl->ready_for_splicing_count++;
+  if (likely(q->len > 4)) afl->ready_for_splicing_count++;
 
   ++afl->queued_paths;
   ++afl->pending_not_fuzzed;
-- 
cgit v1.2.3


From e87eca7fe8ec3ed0ba79e7722350ad502b67218b Mon Sep 17 00:00:00 2001
From: Marcel Boehme <marcel.boehme@acm.org>
Date: Tue, 29 Sep 2020 11:53:27 +0000
Subject: Patching and improving AFLFast schedules.

---
 src/afl-fuzz-queue.c | 84 +++++++++++++++++++++++++++++++++-------------------
 1 file changed, 54 insertions(+), 30 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 53c3e984..dfabba7b 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -25,6 +25,7 @@
 #include "afl-fuzz.h"
 #include <limits.h>
 #include <ctype.h>
+#include <math.h>
 
 /* Mark deterministic checks as done for a particular queue entry. We use the
    .state file to avoid repeating deterministic fuzzing when resuming aborted
@@ -218,7 +219,6 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
   q->len = len;
   q->depth = afl->cur_depth + 1;
   q->passed_det = passed_det;
-  q->n_fuzz = 1;
   q->trace_mini = NULL;
 
   if (q->depth > afl->max_depth) { afl->max_depth = q->depth; }
@@ -307,8 +307,10 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) {
   u64 fav_factor;
   u64 fuzz_p2;
 
-  if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE))
-    fuzz_p2 = next_pow2(q->n_fuzz);
+  if (unlikely(afl->schedule >= FAST && afl->schedule < RARE))
+    fuzz_p2 = 0; // Skip the fuzz_p2 comparison 
+  else if (unlikely(afl->schedule == RARE))
+    fuzz_p2 = next_pow2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]);
   else
     fuzz_p2 = q->fuzz_level;
 
@@ -334,7 +336,7 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) {
         u64 top_rated_fav_factor;
         u64 top_rated_fuzz_p2;
         if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE))
-          top_rated_fuzz_p2 = next_pow2(afl->top_rated[i]->n_fuzz);
+          top_rated_fuzz_p2 = next_pow2(afl->n_fuzz[afl->top_rated[i]->exec_cksum % n_fuzz_size]);
         else
           top_rated_fuzz_p2 = afl->top_rated[i]->fuzz_level;
 
@@ -605,11 +607,10 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
   }
 
-  u64 fuzz = q->n_fuzz;
-  u64 fuzz_total;
+  u32 n_paths;
+  double factor = 1.0;
+  long double fuzz_mu;
 
-  u32 n_paths, fuzz_mu;
-  u32 factor = 1;
 
   switch (afl->schedule) {
 
@@ -624,60 +625,83 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
       break;
 
     case COE:
-      fuzz_total = 0;
+      fuzz_mu = 0.0;
       n_paths = 0;
 
+      // Don't modify perf_score for unfuzzed seeds
+      if (q->fuzz_level == 0) break;
+
       struct queue_entry *queue_it = afl->queue;
       while (queue_it) {
 
-        fuzz_total += queue_it->n_fuzz;
+        fuzz_mu += log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]);
         n_paths++;
+
         queue_it = queue_it->next;
 
       }
 
       if (unlikely(!n_paths)) { FATAL("Queue state corrupt"); }
 
-      fuzz_mu = fuzz_total / n_paths;
-      if (fuzz <= fuzz_mu) {
+      fuzz_mu = fuzz_mu / n_paths;
 
-        if (q->fuzz_level < 16) {
+      if (log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]) > fuzz_mu) {
 
-          factor = ((u32)(1 << q->fuzz_level));
+        /* Never skip favourites */
+        if (!q->favored) factor = 0;
 
-        } else {
+        break;
 
-          factor = MAX_FACTOR;
+      }
 
-        }
+    // Fall through
+    case FAST:
 
-      } else {
+      // Don't modify unfuzzed seeds
+      if (q->fuzz_level == 0) break;
 
-        factor = 0;
+      switch ((u32)log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size])) {
 
-      }
+        case 0 ... 1:
+          factor = 4;
+          break;
 
-      break;
+        case 2 ... 3:
+          factor = 3;
+          break;
 
-    case FAST:
-      if (q->fuzz_level < 16) {
+        case 4:
+          factor = 2;
+          break;
 
-        factor = ((u32)(1 << q->fuzz_level)) / (fuzz == 0 ? 1 : fuzz);
+        case 5:
+          break;
 
-      } else {
+        case 6:
+          if (!q->favored) factor = 0.8;
+          break;
 
-        factor = MAX_FACTOR / (fuzz == 0 ? 1 : next_pow2(fuzz));
+        case 7:
+          if (!q->favored) factor = 0.6;
+          break;
+
+        default:
+          if (!q->favored) factor = 0.4;
+          break;
 
       }
 
+      if (q->favored)
+        factor *= 1.15;
+
       break;
 
     case LIN:
-      factor = q->fuzz_level / (fuzz == 0 ? 1 : fuzz);
+      factor = q->fuzz_level / (afl->n_fuzz[q->exec_cksum % n_fuzz_size] + 1);
       break;
 
     case QUAD:
-      factor = q->fuzz_level * q->fuzz_level / (fuzz == 0 ? 1 : fuzz);
+      factor = q->fuzz_level * q->fuzz_level / (afl->n_fuzz[q->exec_cksum % n_fuzz_size] + 1);
       break;
 
     case MMOPT:
@@ -703,7 +727,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
       // the more often fuzz result paths are equal to this queue entry,
       // reduce its value
       perf_score *=
-          (1 - (double)((double)q->n_fuzz / (double)afl->fsrv.total_execs));
+          (1 - (double)((double)afl->n_fuzz[q->exec_cksum % n_fuzz_size] / (double)afl->fsrv.total_execs));
 
       break;
 
@@ -724,7 +748,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
     perf_score *= 2;
 
-  } else if (perf_score < 1) {
+  } else if (afl->schedule != COE && perf_score < 1) {
 
     // Add a lower bound to AFLFast's energy assignment strategies
     perf_score = 1;
-- 
cgit v1.2.3


From 383cd487a2c28012c80341f8517e473120af4d19 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Tue, 29 Sep 2020 15:02:57 +0200
Subject: small improvements to Marcel's patch, fix laf-intel + redqueen
 crashes

---
 src/afl-fuzz-queue.c | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index dfabba7b..0d7d0314 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -308,9 +308,9 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) {
   u64 fuzz_p2;
 
   if (unlikely(afl->schedule >= FAST && afl->schedule < RARE))
-    fuzz_p2 = 0; // Skip the fuzz_p2 comparison 
+    fuzz_p2 = 0;  // Skip the fuzz_p2 comparison
   else if (unlikely(afl->schedule == RARE))
-    fuzz_p2 = next_pow2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]);
+    fuzz_p2 = next_pow2(afl->n_fuzz[q->n_fuzz_entry]);
   else
     fuzz_p2 = q->fuzz_level;
 
@@ -336,7 +336,8 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) {
         u64 top_rated_fav_factor;
         u64 top_rated_fuzz_p2;
         if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE))
-          top_rated_fuzz_p2 = next_pow2(afl->n_fuzz[afl->top_rated[i]->exec_cksum % n_fuzz_size]);
+          top_rated_fuzz_p2 =
+              next_pow2(afl->n_fuzz[afl->top_rated[i]->n_fuzz_entry]);
         else
           top_rated_fuzz_p2 = afl->top_rated[i]->fuzz_level;
 
@@ -607,11 +608,10 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
   }
 
-  u32 n_paths;
-  double factor = 1.0;
+  u32         n_paths;
+  double      factor = 1.0;
   long double fuzz_mu;
 
-
   switch (afl->schedule) {
 
     case EXPLORE:
@@ -634,7 +634,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
       struct queue_entry *queue_it = afl->queue;
       while (queue_it) {
 
-        fuzz_mu += log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]);
+        fuzz_mu += log2(afl->n_fuzz[q->n_fuzz_entry]);
         n_paths++;
 
         queue_it = queue_it->next;
@@ -645,7 +645,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
       fuzz_mu = fuzz_mu / n_paths;
 
-      if (log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]) > fuzz_mu) {
+      if (log2(afl->n_fuzz[q->n_fuzz_entry]) > fuzz_mu) {
 
         /* Never skip favourites */
         if (!q->favored) factor = 0;
@@ -660,7 +660,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
       // Don't modify unfuzzed seeds
       if (q->fuzz_level == 0) break;
 
-      switch ((u32)log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size])) {
+      switch ((u32)log2(afl->n_fuzz[q->n_fuzz_entry])) {
 
         case 0 ... 1:
           factor = 4;
@@ -691,17 +691,17 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
       }
 
-      if (q->favored)
-        factor *= 1.15;
+      if (q->favored) factor *= 1.15;
 
       break;
 
     case LIN:
-      factor = q->fuzz_level / (afl->n_fuzz[q->exec_cksum % n_fuzz_size] + 1);
+      factor = q->fuzz_level / (afl->n_fuzz[q->n_fuzz_entry] + 1);
       break;
 
     case QUAD:
-      factor = q->fuzz_level * q->fuzz_level / (afl->n_fuzz[q->exec_cksum % n_fuzz_size] + 1);
+      factor =
+          q->fuzz_level * q->fuzz_level / (afl->n_fuzz[q->n_fuzz_entry] + 1);
       break;
 
     case MMOPT:
@@ -726,8 +726,8 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
       perf_score += (q->tc_ref * 10);
       // the more often fuzz result paths are equal to this queue entry,
       // reduce its value
-      perf_score *=
-          (1 - (double)((double)afl->n_fuzz[q->exec_cksum % n_fuzz_size] / (double)afl->fsrv.total_execs));
+      perf_score *= (1 - (double)((double)afl->n_fuzz[q->n_fuzz_entry] /
+                                  (double)afl->fsrv.total_execs));
 
       break;
 
-- 
cgit v1.2.3


From a4b60ca5b61c9bca5fa7b67528baeb3a8ea9320e Mon Sep 17 00:00:00 2001
From: Dominik Maier <domenukk@gmail.com>
Date: Tue, 6 Oct 2020 15:37:59 +0200
Subject: testcase cache added

---
 src/afl-fuzz-queue.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 0d7d0314..e2387aaa 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -220,6 +220,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
   q->depth = afl->cur_depth + 1;
   q->passed_det = passed_det;
   q->trace_mini = NULL;
+  q->testcase_buf = NULL;
 
   if (q->depth > afl->max_depth) { afl->max_depth = q->depth; }
 
@@ -767,3 +768,67 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
 }
 
+/* Tell afl that this testcase may be evicted from the cache */
+inline void queue_testcase_release(afl_state_t *afl, struct queue_entry *q) {
+  (void) afl;
+  q->testcase_refs--;
+  if (unlikely(q->testcase_refs < 0)) { FATAL("Testcase refcount smaller than 0"); }
+}
+
+/* Returns the testcase buf from the file behind this queue entry.
+  Increases the refcount. */
+u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) {
+  if (!q->testcase_buf) {
+    u32 tid = 0;
+    /* Buf not cached, let's do that now */
+
+    if (likely(afl->q_testcase_cache_count == TESTCASE_CACHE_SIZE)) {
+      /* Cache full. We neet to evict one to map one.
+      Get a random one which is not in use */
+      do {
+
+          tid = rand_below(afl, afl->q_testcase_cache_count);
+
+      } while (afl->q_testcase_cache[tid]->testcase_refs > 0);
+
+      struct queue_entry *old_cached = afl->q_testcase_cache[tid];
+      /* free the current buf from cache */
+      munmap(old_cached->testcase_buf, old_cached->len);
+      old_cached->testcase_buf = NULL;
+      
+    } else {
+      tid = afl->q_testcase_cache_count;
+      afl->q_testcase_cache_count++;
+    }
+
+    /* Map the test case into memory. */
+
+    int fd = open(q->fname, O_RDONLY);
+
+    if (unlikely(fd < 0)) {
+
+      PFATAL("Unable to open '%s'", q->fname);
+
+    }
+
+    u32 len = q->len;
+
+    q->testcase_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
+
+    if (unlikely(q->testcase_buf == MAP_FAILED)) {
+
+      PFATAL("Unable to mmap '%s' with len %d", q->fname, len);
+
+    }
+
+    close(fd);
+
+    /* Register us as cached */
+    afl->q_testcase_cache[tid] = q;
+
+  }
+  q->testcase_refs++;
+  if (!q->testcase_buf) { FATAL("Testcase buf is NULL, this should never happen"); }
+  return q->testcase_buf;
+
+}
-- 
cgit v1.2.3


From 74dc227c4412d0121c9b972e5d89db89f54c6b3a Mon Sep 17 00:00:00 2001
From: Dominik Maier <domenukk@gmail.com>
Date: Tue, 6 Oct 2020 15:38:36 +0200
Subject: code format

---
 src/afl-fuzz-queue.c | 34 ++++++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 10 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index e2387aaa..721f9ac7 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -770,24 +770,33 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
 /* Tell afl that this testcase may be evicted from the cache */
 inline void queue_testcase_release(afl_state_t *afl, struct queue_entry *q) {
-  (void) afl;
+
+  (void)afl;
   q->testcase_refs--;
-  if (unlikely(q->testcase_refs < 0)) { FATAL("Testcase refcount smaller than 0"); }
+  if (unlikely(q->testcase_refs < 0)) {
+
+    FATAL("Testcase refcount smaller than 0");
+
+  }
+
 }
 
 /* Returns the testcase buf from the file behind this queue entry.
   Increases the refcount. */
 u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) {
+
   if (!q->testcase_buf) {
+
     u32 tid = 0;
     /* Buf not cached, let's do that now */
 
     if (likely(afl->q_testcase_cache_count == TESTCASE_CACHE_SIZE)) {
+
       /* Cache full. We neet to evict one to map one.
       Get a random one which is not in use */
       do {
 
-          tid = rand_below(afl, afl->q_testcase_cache_count);
+        tid = rand_below(afl, afl->q_testcase_cache_count);
 
       } while (afl->q_testcase_cache[tid]->testcase_refs > 0);
 
@@ -795,21 +804,19 @@ u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) {
       /* free the current buf from cache */
       munmap(old_cached->testcase_buf, old_cached->len);
       old_cached->testcase_buf = NULL;
-      
+
     } else {
+
       tid = afl->q_testcase_cache_count;
       afl->q_testcase_cache_count++;
+
     }
 
     /* Map the test case into memory. */
 
     int fd = open(q->fname, O_RDONLY);
 
-    if (unlikely(fd < 0)) {
-
-      PFATAL("Unable to open '%s'", q->fname);
-
-    }
+    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
 
     u32 len = q->len;
 
@@ -827,8 +834,15 @@ u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) {
     afl->q_testcase_cache[tid] = q;
 
   }
+
   q->testcase_refs++;
-  if (!q->testcase_buf) { FATAL("Testcase buf is NULL, this should never happen"); }
+  if (!q->testcase_buf) {
+
+    FATAL("Testcase buf is NULL, this should never happen");
+
+  }
+
   return q->testcase_buf;
 
 }
+
-- 
cgit v1.2.3


From 4f207b4eba26c2b268ba2fd0a51298d6ab88f110 Mon Sep 17 00:00:00 2001
From: Dominik Maier <domenukk@gmail.com>
Date: Tue, 6 Oct 2020 16:20:32 +0200
Subject: fixed ref check

---
 src/afl-fuzz-queue.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 721f9ac7..58e026f5 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -772,13 +772,14 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 inline void queue_testcase_release(afl_state_t *afl, struct queue_entry *q) {
 
   (void)afl;
-  q->testcase_refs--;
-  if (unlikely(q->testcase_refs < 0)) {
+  if (unlikely(q->testcase_refs == 0)) {
 
-    FATAL("Testcase refcount smaller than 0");
+    FATAL("Testcase refcount reduced past 0");
 
   }
 
+  q->testcase_refs--;
+
 }
 
 /* Returns the testcase buf from the file behind this queue entry.
-- 
cgit v1.2.3


From 2d5fadc1e6a684b5e3e527a64b614f6b1ba8415f Mon Sep 17 00:00:00 2001
From: Dominik Maier <domenukk@gmail.com>
Date: Tue, 6 Oct 2020 16:45:25 +0200
Subject: hunting ref underflow

---
 src/afl-fuzz-queue.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 58e026f5..0b491202 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -837,10 +837,17 @@ u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) {
   }
 
   q->testcase_refs++;
-  if (!q->testcase_buf) {
+  if (unlikely(!q->testcase_buf || !q->testcase_refs)) {
+    if (!q->testcase_buf) {
+
+      FATAL("Testcase buf is NULL, this should never happen");
 
-    FATAL("Testcase buf is NULL, this should never happen");
+    }
+    if (!q->testcase_refs) {
 
+      FATAL("Testcase ref overflow. Missing a testcase release somwhere?");
+
+    }
   }
 
   return q->testcase_buf;
-- 
cgit v1.2.3


From 6a397d6111a21ebbf736237609c1c69d47c40f03 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Sun, 11 Oct 2020 14:31:31 +0200
Subject: add new seed selection algo and make it the default

---
 src/afl-fuzz-queue.c | 124 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 124 insertions(+)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 0d7d0314..d608e890 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -27,6 +27,129 @@
 #include <ctype.h>
 #include <math.h>
 
+inline u32 select_next_queue_entry(afl_state_t *afl) {
+
+  u32 r = rand_below(afl, 0xffffffff);
+  u32 s = r % afl->queued_paths;
+  // fprintf(stderr, "select: r=%u s=%u ... r < prob[s]=%f ? s=%u :
+  // alias[%u]=%u\n", r, s, afl->alias_probability[s], s, s,
+  // afl->alias_table[s]);
+  return (r < afl->alias_probability[s] ? s : afl->alias_table[s]);
+
+}
+
+void create_alias_table(afl_state_t *afl) {
+
+  u32 n = afl->queued_paths, i = 0, a, g;
+
+  afl->alias_table =
+      (u32 *)afl_realloc((void **)&afl->alias_table, n * sizeof(u32));
+  afl->alias_probability = (double *)afl_realloc(
+      (void **)&afl->alias_probability, n * sizeof(double));
+  double *P = (double *)afl_realloc(AFL_BUF_PARAM(out), n * sizeof(double));
+  int *   S = (u32 *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32));
+  int *   L = (u32 *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32));
+
+  if (!P || !S || !L) FATAL("could not aquire memory for alias table");
+  memset((void *)afl->alias_table, 0, n * sizeof(u32));
+  memset((void *)afl->alias_probability, 0, n * sizeof(double));
+
+  double sum = 0;
+
+  for (i = 0; i < n; i++) {
+
+    struct queue_entry *q = afl->queue_buf[i];
+
+    if (!q->disabled) q->perf_score = calculate_score(afl, q);
+
+    sum += q->perf_score;
+    /*
+        if (afl->debug)
+          fprintf(stderr, "entry %u: score=%f %s (sum: %f)\n", i, q->perf_score,
+                  q->disabled ? "disabled" : "", sum);
+    */
+
+  }
+
+  for (i = 0; i < n; i++) {
+
+    struct queue_entry *q = afl->queue_buf[i];
+
+    P[i] = q->perf_score * n / sum;
+
+  }
+
+  int nS = 0, nL = 0, s;
+  for (s = (s32)n - 1; s >= 0; --s) {
+
+    if (P[s] < 1)
+      S[nS++] = s;
+    else
+      L[nL++] = s;
+
+  }
+
+  while (nS && nL) {
+
+    a = S[--nS];
+    g = L[--nL];
+    afl->alias_probability[a] = P[a];
+    afl->alias_table[a] = g;
+    P[g] = P[g] + P[a] - 1;
+    if (P[g] < 1)
+      S[nS++] = g;
+    else
+      L[nL++] = g;
+
+  }
+
+  while (nL)
+    afl->alias_probability[L[--nL]] = 1;
+
+  while (nS)
+    afl->alias_probability[S[--nS]] = 1;
+
+  /*
+    if (afl->debug) {
+
+      fprintf(stderr, "  %-3s  %-3s  %-9s\n", "entry", "alias", "prob");
+      for (u32 i = 0; i < n; ++i)
+        fprintf(stderr, "  %3i  %3i  %9.7f\n", i, afl->alias_table[i],
+                afl->alias_probability[i]);
+
+    }
+
+    int prob = 0;
+    fprintf(stderr, "Alias:");
+    for (i = 0; i < n; i++) {
+
+      fprintf(stderr, " [%u]=%u", i, afl->alias_table[i]);
+      if (afl->alias_table[i] >= n)
+        prob = i;
+
+    }
+
+    fprintf(stderr, "\n");
+
+    if (prob) {
+
+      fprintf(stderr, "PROBLEM! alias[%u] = %u\n", prob,
+    afl->alias_table[prob]);
+
+      for (i = 0; i < n; i++) {
+
+        struct queue_entry *q = afl->queue_buf[i];
+
+        fprintf(stderr, "%u: score=%f\n", i, q->perf_score);
+
+      }
+
+    }
+
+  */
+
+}
+
 /* Mark deterministic checks as done for a particular queue entry. We use the
    .state file to avoid repeating deterministic fuzzing when resuming aborted
    scans. */
@@ -237,6 +360,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
   if (likely(q->len > 4)) afl->ready_for_splicing_count++;
 
   ++afl->queued_paths;
+  ++afl->active_paths;
   ++afl->pending_not_fuzzed;
 
   afl->cycles_wo_finds = 0;
-- 
cgit v1.2.3


From b7e0490bcdaa7fa792a9dccfa5983e03af92730e Mon Sep 17 00:00:00 2001
From: vanhauser-thc <vh@thc.org>
Date: Mon, 12 Oct 2020 03:44:34 +0200
Subject: Revert "Merge branch 'memcache_marc' into dev"

This reverts commit c03fbcedaa68db5324423975a34331287426f7c2, reversing
changes made to dab017dddaaab6d836a590f7bba3eea3549758d2.
---
 src/afl-fuzz-queue.c | 87 ----------------------------------------------------
 1 file changed, 87 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index a034b168..d608e890 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -343,7 +343,6 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
   q->depth = afl->cur_depth + 1;
   q->passed_det = passed_det;
   q->trace_mini = NULL;
-  q->testcase_buf = NULL;
 
   if (q->depth > afl->max_depth) { afl->max_depth = q->depth; }
 
@@ -892,89 +891,3 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
 }
 
-/* Tell afl that this testcase may be evicted from the cache */
-inline void queue_testcase_release(afl_state_t *afl, struct queue_entry *q) {
-
-  (void)afl;
-  if (unlikely(q->testcase_refs == 0)) {
-
-    FATAL("Testcase refcount reduced past 0");
-
-  }
-
-  q->testcase_refs--;
-
-}
-
-/* Returns the testcase buf from the file behind this queue entry.
-  Increases the refcount. */
-u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) {
-
-  if (!q->testcase_buf) {
-
-    u32 tid = 0;
-    /* Buf not cached, let's do that now */
-
-    if (likely(afl->q_testcase_cache_count == TESTCASE_CACHE_SIZE)) {
-
-      /* Cache full. We neet to evict one to map one.
-      Get a random one which is not in use */
-      do {
-
-        tid = rand_below(afl, afl->q_testcase_cache_count);
-
-      } while (afl->q_testcase_cache[tid]->testcase_refs > 0);
-
-      struct queue_entry *old_cached = afl->q_testcase_cache[tid];
-      /* free the current buf from cache */
-      munmap(old_cached->testcase_buf, old_cached->len);
-      old_cached->testcase_buf = NULL;
-
-    } else {
-
-      tid = afl->q_testcase_cache_count;
-      afl->q_testcase_cache_count++;
-
-    }
-
-    /* Map the test case into memory. */
-
-    int fd = open(q->fname, O_RDONLY);
-
-    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
-
-    u32 len = q->len;
-
-    q->testcase_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
-
-    if (unlikely(q->testcase_buf == MAP_FAILED)) {
-
-      PFATAL("Unable to mmap '%s' with len %d", q->fname, len);
-
-    }
-
-    close(fd);
-
-    /* Register us as cached */
-    afl->q_testcase_cache[tid] = q;
-
-  }
-
-  q->testcase_refs++;
-  if (unlikely(!q->testcase_buf || !q->testcase_refs)) {
-    if (!q->testcase_buf) {
-
-      FATAL("Testcase buf is NULL, this should never happen");
-
-    }
-    if (!q->testcase_refs) {
-
-      FATAL("Testcase ref overflow. Missing a testcase release somwhere?");
-
-    }
-  }
-
-  return q->testcase_buf;
-
-}
-
-- 
cgit v1.2.3


From d9b63766dfdb8feeb1dc6f7c51c17abf07ee4086 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Mon, 12 Oct 2020 11:12:16 +0200
Subject: fix new seed selection algo

---
 src/afl-fuzz-queue.c | 59 +++++++++++++---------------------------------------
 1 file changed, 14 insertions(+), 45 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index d608e890..f224d851 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -27,17 +27,22 @@
 #include <ctype.h>
 #include <math.h>
 
+/* select next queue entry based on alias algo - fast! */
+
 inline u32 select_next_queue_entry(afl_state_t *afl) {
 
-  u32 r = rand_below(afl, 0xffffffff);
-  u32 s = r % afl->queued_paths;
-  // fprintf(stderr, "select: r=%u s=%u ... r < prob[s]=%f ? s=%u :
-  // alias[%u]=%u\n", r, s, afl->alias_probability[s], s, s,
-  // afl->alias_table[s]);
-  return (r < afl->alias_probability[s] ? s : afl->alias_table[s]);
+  u32 s = rand_below(afl, afl->queued_paths);
+  double p = rand_next_percent(afl);
+  /*
+  fprintf(stderr, "select: p=%f s=%u ... p < prob[s]=%f ? s=%u : alias[%u]=%u"
+  " ==> %u\n", p, s, afl->alias_probability[s], s, s, afl->alias_table[s], p < afl->alias_probability[s] ? s : afl->alias_table[s]);
+  */
+  return (p < afl->alias_probability[s] ? s : afl->alias_table[s]);
 
 }
 
+/* create the alias table that allows weighted random selection - expensive */
+
 void create_alias_table(afl_state_t *afl) {
 
   u32 n = afl->queued_paths, i = 0, a, g;
@@ -63,11 +68,6 @@ void create_alias_table(afl_state_t *afl) {
     if (!q->disabled) q->perf_score = calculate_score(afl, q);
 
     sum += q->perf_score;
-    /*
-        if (afl->debug)
-          fprintf(stderr, "entry %u: score=%f %s (sum: %f)\n", i, q->perf_score,
-                  q->disabled ? "disabled" : "", sum);
-    */
 
   }
 
@@ -110,41 +110,10 @@ void create_alias_table(afl_state_t *afl) {
     afl->alias_probability[S[--nS]] = 1;
 
   /*
-    if (afl->debug) {
-
-      fprintf(stderr, "  %-3s  %-3s  %-9s\n", "entry", "alias", "prob");
+      fprintf(stderr, "  %-3s  %-3s  %-9s  %-9s\n", "entry", "alias", "prob", "perf");
       for (u32 i = 0; i < n; ++i)
-        fprintf(stderr, "  %3i  %3i  %9.7f\n", i, afl->alias_table[i],
-                afl->alias_probability[i]);
-
-    }
-
-    int prob = 0;
-    fprintf(stderr, "Alias:");
-    for (i = 0; i < n; i++) {
-
-      fprintf(stderr, " [%u]=%u", i, afl->alias_table[i]);
-      if (afl->alias_table[i] >= n)
-        prob = i;
-
-    }
-
-    fprintf(stderr, "\n");
-
-    if (prob) {
-
-      fprintf(stderr, "PROBLEM! alias[%u] = %u\n", prob,
-    afl->alias_table[prob]);
-
-      for (i = 0; i < n; i++) {
-
-        struct queue_entry *q = afl->queue_buf[i];
-
-        fprintf(stderr, "%u: score=%f\n", i, q->perf_score);
-
-      }
-
-    }
+        fprintf(stderr, "  %3i  %3i  %9.7f  %9.7f\n", i, afl->alias_table[i],
+                afl->alias_probability[i], afl->queue_buf[i]->perf_score);
 
   */
 
-- 
cgit v1.2.3


From 56ac3fcdc511d124ad058412021ead21bbbcf4bf Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Wed, 14 Oct 2020 15:30:30 +0200
Subject: configurable testcache with malloc (#581)

* cache item number to cache memory size

* reload testcase if trimming changed the size

* fix splicing selection

* slim splicing

* import sync fix

* write testcache stats to fuzzer_stats

* fix new seed selection algo

* malloc+read instead of mmap

* fix

* testcache is configurable now and no reference counts

* fixes compilation, test script

* fixes

* switch TEST_CC to afl-cc in makefile

* code format

* fix

* fix crash

* fix crash

* fix env help output

* remove unnecessary pointer resets

* fix endless loop bug

* actually use the cache if set

* one more fix

* increase default cache entries, add default cache size value to config.h

Co-authored-by: hexcoder- <heiko@hexco.de>
---
 src/afl-fuzz-queue.c | 167 ++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 152 insertions(+), 15 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index f224d851..c634328f 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -31,11 +31,12 @@
 
 inline u32 select_next_queue_entry(afl_state_t *afl) {
 
-  u32 s = rand_below(afl, afl->queued_paths);
+  u32    s = rand_below(afl, afl->queued_paths);
   double p = rand_next_percent(afl);
   /*
   fprintf(stderr, "select: p=%f s=%u ... p < prob[s]=%f ? s=%u : alias[%u]=%u"
-  " ==> %u\n", p, s, afl->alias_probability[s], s, s, afl->alias_table[s], p < afl->alias_probability[s] ? s : afl->alias_table[s]);
+  " ==> %u\n", p, s, afl->alias_probability[s], s, s, afl->alias_table[s], p <
+  afl->alias_probability[s] ? s : afl->alias_table[s]);
   */
   return (p < afl->alias_probability[s] ? s : afl->alias_table[s]);
 
@@ -55,7 +56,7 @@ void create_alias_table(afl_state_t *afl) {
   int *   S = (u32 *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32));
   int *   L = (u32 *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32));
 
-  if (!P || !S || !L) FATAL("could not aquire memory for alias table");
+  if (!P || !S || !L) { FATAL("could not aquire memory for alias table"); }
   memset((void *)afl->alias_table, 0, n * sizeof(u32));
   memset((void *)afl->alias_probability, 0, n * sizeof(double));
 
@@ -65,7 +66,7 @@ void create_alias_table(afl_state_t *afl) {
 
     struct queue_entry *q = afl->queue_buf[i];
 
-    if (!q->disabled) q->perf_score = calculate_score(afl, q);
+    if (!q->disabled) { q->perf_score = calculate_score(afl, q); }
 
     sum += q->perf_score;
 
@@ -74,19 +75,23 @@ void create_alias_table(afl_state_t *afl) {
   for (i = 0; i < n; i++) {
 
     struct queue_entry *q = afl->queue_buf[i];
-
-    P[i] = q->perf_score * n / sum;
+    P[i] = (q->perf_score * n) / sum;
 
   }
 
   int nS = 0, nL = 0, s;
   for (s = (s32)n - 1; s >= 0; --s) {
 
-    if (P[s] < 1)
+    if (P[s] < 1) {
+
       S[nS++] = s;
-    else
+
+    } else {
+
       L[nL++] = s;
 
+    }
+
   }
 
   while (nS && nL) {
@@ -96,11 +101,16 @@ void create_alias_table(afl_state_t *afl) {
     afl->alias_probability[a] = P[a];
     afl->alias_table[a] = g;
     P[g] = P[g] + P[a] - 1;
-    if (P[g] < 1)
+    if (P[g] < 1) {
+
       S[nS++] = g;
-    else
+
+    } else {
+
       L[nL++] = g;
 
+    }
+
   }
 
   while (nL)
@@ -110,11 +120,10 @@ void create_alias_table(afl_state_t *afl) {
     afl->alias_probability[S[--nS]] = 1;
 
   /*
-      fprintf(stderr, "  %-3s  %-3s  %-9s  %-9s\n", "entry", "alias", "prob", "perf");
-      for (u32 i = 0; i < n; ++i)
-        fprintf(stderr, "  %3i  %3i  %9.7f  %9.7f\n", i, afl->alias_table[i],
-                afl->alias_probability[i], afl->queue_buf[i]->perf_score);
-
+  fprintf(stderr, "  entry  alias  probability  perf_score\n");
+  for (u32 i = 0; i < n; ++i)
+    fprintf(stderr, "  %5u  %5u  %11u  %0.9f\n", i, afl->alias_table[i],
+            afl->alias_probability[i], afl->queue_buf[i]->perf_score);
   */
 
 }
@@ -860,3 +869,131 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
 }
 
+void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q,
+                           u32 old_len) {
+
+  if (likely(q->testcase_buf)) {
+
+    free(q->testcase_buf);
+    int fd = open(q->fname, O_RDONLY);
+
+    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
+
+    u32 len = q->len;
+    q->testcase_buf = malloc(len);
+
+    if (unlikely(!q->testcase_buf)) {
+
+      PFATAL("Unable to mmap '%s' with len %d", q->fname, len);
+
+    }
+
+    close(fd);
+    afl->q_testcase_cache_size = afl->q_testcase_cache_size + q->len - old_len;
+
+  }
+
+}
+
+/* Returns the testcase buf from the file behind this queue entry.
+  Increases the refcount. */
+inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
+
+  u32 len = q->len;
+
+  /* first handle if no testcase cache is configured */
+
+  if (unlikely(!afl->q_testcase_max_cache_size)) {
+
+    u8 *buf;
+
+    if (q == afl->queue_cur) {
+
+      buf = afl_realloc((void **)&afl->testcase_buf, len);
+
+    } else {
+
+      buf = afl_realloc((void **)&afl->splicecase_buf, len);
+
+    }
+
+    if (unlikely(!buf)) {
+
+      PFATAL("Unable to malloc '%s' with len %u", q->fname, len);
+
+    }
+
+    int fd = open(q->fname, O_RDONLY);
+
+    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
+
+    ck_read(fd, buf, len, q->fname);
+    close(fd);
+    return buf;
+
+  }
+
+  /* now handle the testcase cache */
+
+  if (unlikely(!q->testcase_buf)) {
+
+    /* Buf not cached, let's load it */
+    u32 tid = 0;
+
+    while (unlikely(afl->q_testcase_cache_size + len >=
+                        afl->q_testcase_max_cache_size ||
+                    afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) {
+
+      /* Cache full. We neet to evict one to map one.
+         Get a random one which is not in use */
+
+      do {
+
+        tid = rand_below(afl, afl->q_testcase_max_cache_count);
+
+      } while (afl->q_testcase_cache[tid] == NULL ||
+
+               afl->q_testcase_cache[tid] == afl->queue_cur);
+
+      struct queue_entry *old_cached = afl->q_testcase_cache[tid];
+      free(old_cached->testcase_buf);
+      old_cached->testcase_buf = NULL;
+      afl->q_testcase_cache_size -= old_cached->len;
+      afl->q_testcase_cache[tid] = NULL;
+      --afl->q_testcase_cache_count;
+
+    }
+
+    while (likely(afl->q_testcase_cache[tid] != NULL))
+      ++tid;
+
+    /* Map the test case into memory. */
+
+    int fd = open(q->fname, O_RDONLY);
+
+    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
+
+    q->testcase_buf = malloc(len);
+
+    if (unlikely(!q->testcase_buf)) {
+
+      PFATAL("Unable to malloc '%s' with len %u", q->fname, len);
+
+    }
+
+    ck_read(fd, q->testcase_buf, len, q->fname);
+    close(fd);
+
+    /* Register testcase as cached */
+    afl->q_testcase_cache[tid] = q;
+    afl->q_testcase_cache_size += q->len;
+    ++afl->q_testcase_cache_count;
+    if (tid >= afl->q_testcase_max_cache_count)
+      afl->q_testcase_max_cache_count = tid + 1;
+
+  }
+
+  return q->testcase_buf;
+
+}
+
-- 
cgit v1.2.3


From 0f8529a3db242131486cc3bf4a66c024c2b3e126 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Thu, 15 Oct 2020 15:22:40 +0200
Subject: prepare halloween fuzzbench run

---
 src/afl-fuzz-queue.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index c634328f..38d7f77e 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -120,10 +120,11 @@ void create_alias_table(afl_state_t *afl) {
     afl->alias_probability[S[--nS]] = 1;
 
   /*
-  fprintf(stderr, "  entry  alias  probability  perf_score\n");
+  fprintf(stderr, "  entry  alias  probability  perf_score   filename\n");
   for (u32 i = 0; i < n; ++i)
-    fprintf(stderr, "  %5u  %5u  %11u  %0.9f\n", i, afl->alias_table[i],
-            afl->alias_probability[i], afl->queue_buf[i]->perf_score);
+    fprintf(stderr, "  %5u  %5u  %11u  %0.9f  %s\n", i, afl->alias_table[i],
+            afl->alias_probability[i], afl->queue_buf[i]->perf_score,
+            afl->queue_buf[i]->fname);
   */
 
 }
-- 
cgit v1.2.3


From 354bda28465588e424c0a93b413af01a603191ce Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Thu, 15 Oct 2020 15:33:47 +0200
Subject: fix reget of testcase after trim

---
 src/afl-fuzz-queue.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 38d7f77e..095a391f 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -875,22 +875,27 @@ void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q,
 
   if (likely(q->testcase_buf)) {
 
-    free(q->testcase_buf);
-    int fd = open(q->fname, O_RDONLY);
+    u32 len = q->len;
 
-    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
+    if (len != old_len) {
 
-    u32 len = q->len;
-    q->testcase_buf = malloc(len);
+      afl->q_testcase_cache_size =
+          afl->q_testcase_cache_size + q->len - old_len;
+      q->testcase_buf = realloc(q->testcase_buf, len);
+      if (unlikely(!q->testcase_buf)) {
 
-    if (unlikely(!q->testcase_buf)) {
+        PFATAL("Unable to malloc '%s' with len %d", q->fname, len);
 
-      PFATAL("Unable to mmap '%s' with len %d", q->fname, len);
+      }
 
     }
 
+    int fd = open(q->fname, O_RDONLY);
+
+    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
+
+    ck_read(fd, q->testcase_buf, len, q->fname);
     close(fd);
-    afl->q_testcase_cache_size = afl->q_testcase_cache_size + q->len - old_len;
 
   }
 
-- 
cgit v1.2.3


From f41aafa4f7aa446c3cb1cbe6d77364cf32a6c6cb Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Thu, 15 Oct 2020 15:48:39 +0200
Subject: retake from mem if possible

---
 src/afl-fuzz-queue.c | 39 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 5 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 095a391f..92b722f6 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -870,8 +870,10 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) {
 
 }
 
-void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q,
-                           u32 old_len) {
+/* after a custom trim we need to reload the testcase from disk */
+
+inline void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q,
+                                  u32 old_len) {
 
   if (likely(q->testcase_buf)) {
 
@@ -879,9 +881,9 @@ void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q,
 
     if (len != old_len) {
 
-      afl->q_testcase_cache_size =
-          afl->q_testcase_cache_size + q->len - old_len;
+      afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len;
       q->testcase_buf = realloc(q->testcase_buf, len);
+
       if (unlikely(!q->testcase_buf)) {
 
         PFATAL("Unable to malloc '%s' with len %d", q->fname, len);
@@ -901,8 +903,35 @@ void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q,
 
 }
 
+/* after a normal trim we need to replace the testcase with the new data */
+
+inline void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q,
+                                      u8 *in, u32 len, u32 old_len) {
+
+  if (likely(q->testcase_buf)) {
+
+    if (len != old_len) {
+
+      afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len;
+      q->testcase_buf = realloc(q->testcase_buf, len);
+
+      if (unlikely(!q->testcase_buf)) {
+
+        PFATAL("Unable to malloc '%s' with len %d", q->fname, len);
+
+      }
+
+    }
+
+    memcpy(q->testcase_buf, in, len);
+
+  }
+
+}
+
 /* Returns the testcase buf from the file behind this queue entry.
   Increases the refcount. */
+
 inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
   u32 len = q->len;
@@ -913,7 +942,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
     u8 *buf;
 
-    if (q == afl->queue_cur) {
+    if (unlikely(q == afl->queue_cur)) {
 
       buf = afl_realloc((void **)&afl->testcase_buf, len);
 
-- 
cgit v1.2.3


From d5c3b4bafdae8a68e7f63c0afdd1cc5820636f2d Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Fri, 16 Oct 2020 09:35:35 +0200
Subject: directly add new queue to cache

---
 src/afl-fuzz-queue.c | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 92b722f6..f8034ebd 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -1021,7 +1021,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
     /* Register testcase as cached */
     afl->q_testcase_cache[tid] = q;
-    afl->q_testcase_cache_size += q->len;
+    afl->q_testcase_cache_size += len;
     ++afl->q_testcase_cache_count;
     if (tid >= afl->q_testcase_max_cache_count)
       afl->q_testcase_max_cache_count = tid + 1;
@@ -1032,3 +1032,44 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
 }
 
+/* Adds the new queue entry to the cache. */
+
+inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q,
+                                     u8 *mem) {
+
+  u32 len = q->len;
+
+  if (unlikely(afl->q_testcase_cache_size + len >=
+                   afl->q_testcase_max_cache_size ||
+               afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) {
+
+    return;
+
+  }
+
+  u32 tid = 0;
+
+  while (likely(afl->q_testcase_cache[tid] != NULL))
+    ++tid;
+
+  /* Map the test case into memory. */
+
+  q->testcase_buf = malloc(len);
+
+  if (unlikely(!q->testcase_buf)) {
+
+    PFATAL("Unable to malloc '%s' with len %u", q->fname, len);
+
+  }
+
+  memcpy(q->testcase_buf, mem, len);
+
+  /* Register testcase as cached */
+  afl->q_testcase_cache[tid] = q;
+  afl->q_testcase_cache_size += len;
+  ++afl->q_testcase_cache_count;
+  if (tid >= afl->q_testcase_max_cache_count)
+    afl->q_testcase_max_cache_count = tid + 1;
+
+}
+
-- 
cgit v1.2.3


From fcea01a8ea7ec507b675d839035eb0fed2f06867 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Sat, 17 Oct 2020 11:38:55 +0200
Subject: add eviction stat

---
 src/afl-fuzz-queue.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index f8034ebd..e0df7206 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -996,6 +996,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
       afl->q_testcase_cache_size -= old_cached->len;
       afl->q_testcase_cache[tid] = NULL;
       --afl->q_testcase_cache_count;
+      ++afl->q_testcase_evictions;
 
     }
 
-- 
cgit v1.2.3


From ac1c3b87015dd2c9b1bae0198f7925816aa63aec Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Mon, 19 Oct 2020 11:34:57 +0200
Subject: mini improvements

---
 src/afl-fuzz-queue.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index e0df7206..7f157121 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -179,9 +179,9 @@ void mark_as_variable(afl_state_t *afl, struct queue_entry *q) {
 
 void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) {
 
-  u8 fn[PATH_MAX];
+  if (likely(state == q->fs_redundant)) { return; }
 
-  if (state == q->fs_redundant) { return; }
+  u8 fn[PATH_MAX];
 
   q->fs_redundant = state;
 
@@ -521,13 +521,13 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) {
 
 void cull_queue(afl_state_t *afl) {
 
+  if (likely(!afl->score_changed || afl->non_instrumented_mode)) { return; }
+
   struct queue_entry *q;
   u32                 len = (afl->fsrv.map_size >> 3);
   u32                 i;
   u8 *                temp_v = afl->map_tmp_buf;
 
-  if (afl->non_instrumented_mode || !afl->score_changed) { return; }
-
   afl->score_changed = 0;
 
   memset(temp_v, 255, len);
-- 
cgit v1.2.3


From d0cdbc48aebf88bc0cc519db553ca762f794296e Mon Sep 17 00:00:00 2001
From: hexcoder- <heiko@hexco.de>
Date: Tue, 20 Oct 2020 00:07:40 +0200
Subject: always close file descriptor when opened, use standard types

---
 src/afl-fuzz-queue.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 7f157121..1bda0835 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -211,13 +211,16 @@ static u8 check_if_text(struct queue_entry *q) {
   if (q->len < AFL_TXT_MIN_LEN) return 0;
 
   u8  buf[MAX_FILE];
-  s32 fd, len = q->len, offset = 0, ascii = 0, utf8 = 0, comp;
+  int fd;
+  u32 len = q->len, offset = 0, ascii = 0, utf8 = 0;
+  ssize_t comp;
 
   if (len >= MAX_FILE) len = MAX_FILE - 1;
   if ((fd = open(q->fname, O_RDONLY)) < 0) return 0;
-  if ((comp = read(fd, buf, len)) != len) return 0;
-  buf[len] = 0;
+  comp = read(fd, buf, len);
   close(fd);
+  if (comp != (ssize_t)len) return 0;
+  buf[len] = 0;
 
   while (offset < len) {
 
-- 
cgit v1.2.3


From 8d75c089384a0975e39e54f3ba23c37a6101b61c Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Thu, 22 Oct 2020 15:48:09 +0200
Subject: change queue cache algo

---
 src/afl-fuzz-queue.c | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 1bda0835..3e6deb0c 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -210,9 +210,9 @@ static u8 check_if_text(struct queue_entry *q) {
 
   if (q->len < AFL_TXT_MIN_LEN) return 0;
 
-  u8  buf[MAX_FILE];
-  int fd;
-  u32 len = q->len, offset = 0, ascii = 0, utf8 = 0;
+  u8      buf[MAX_FILE];
+  int     fd;
+  u32     len = q->len, offset = 0, ascii = 0, utf8 = 0;
   ssize_t comp;
 
   if (len >= MAX_FILE) len = MAX_FILE - 1;
@@ -913,7 +913,7 @@ inline void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q,
 
   if (likely(q->testcase_buf)) {
 
-    if (len != old_len) {
+    if (likely(len != old_len)) {
 
       afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len;
       q->testcase_buf = realloc(q->testcase_buf, len);
@@ -976,17 +976,21 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
   if (unlikely(!q->testcase_buf)) {
 
     /* Buf not cached, let's load it */
-    u32 tid = 0;
+    u32 tid = afl->q_testcase_max_cache_count;
 
     while (unlikely(afl->q_testcase_cache_size + len >=
                         afl->q_testcase_max_cache_size ||
                     afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) {
 
-      /* Cache full. We neet to evict one to map one.
+      /* Cache full. We neet to evict one or more to map one.
          Get a random one which is not in use */
 
       do {
 
+        // if the cache (MB) is not enough for the queue then this gets
+        // undesirable because q_testcase_max_cache_count grows sometimes
+        // although the number of items in the cache will not change hence
+        // more and more loops
         tid = rand_below(afl, afl->q_testcase_max_cache_count);
 
       } while (afl->q_testcase_cache[tid] == NULL ||
@@ -1003,7 +1007,16 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
     }
 
-    while (likely(afl->q_testcase_cache[tid] != NULL))
+    if (tid >= TESTCASE_ENTRIES) {
+
+      // uh we were full, so now we have to search from start
+      tid = 0;
+
+    }
+
+    // we need this while loop in case there were ever previous evictions but
+    // not in this call.
+    while (unlikely(afl->q_testcase_cache[tid] != NULL))
       ++tid;
 
     /* Map the test case into memory. */
@@ -1047,6 +1060,7 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q,
                    afl->q_testcase_max_cache_size ||
                afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) {
 
+    // no space? will be loaded regularly later.
     return;
 
   }
-- 
cgit v1.2.3


From c866aef37fcf799506d93b9a47d4eb2b77c75f5b Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Thu, 22 Oct 2020 21:07:37 +0200
Subject: maybe enhancement to cache algo

---
 src/afl-fuzz-queue.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 3e6deb0c..db387c33 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -1004,13 +1004,15 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
       afl->q_testcase_cache[tid] = NULL;
       --afl->q_testcase_cache_count;
       ++afl->q_testcase_evictions;
+      if (tid < afl->q_testcase_smallest_free)
+        afl->q_testcase_smallest_free = tid;
 
     }
 
-    if (tid >= TESTCASE_ENTRIES) {
+    if (unlikely(tid >= TESTCASE_ENTRIES)) {
 
       // uh we were full, so now we have to search from start
-      tid = 0;
+      tid = afl->q_testcase_smallest_free;
 
     }
 
@@ -1042,6 +1044,8 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
     ++afl->q_testcase_cache_count;
     if (tid >= afl->q_testcase_max_cache_count)
       afl->q_testcase_max_cache_count = tid + 1;
+    if (tid == afl->q_testcase_smallest_free)
+      afl->q_testcase_smallest_free = tid + 1;
 
   }
 
-- 
cgit v1.2.3


From 0e748ccda713708de6a501d23a58788aba9d0b03 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Fri, 23 Oct 2020 14:05:34 +0200
Subject: set max testcache entries automated if not specified by the user

---
 src/afl-fuzz-queue.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index db387c33..baa80e61 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -978,9 +978,9 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
     /* Buf not cached, let's load it */
     u32 tid = afl->q_testcase_max_cache_count;
 
-    while (unlikely(afl->q_testcase_cache_size + len >=
-                        afl->q_testcase_max_cache_size ||
-                    afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) {
+    while (unlikely(
+        afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size ||
+        afl->q_testcase_cache_count >= afl->q_testcase_max_cache_entries - 1)) {
 
       /* Cache full. We neet to evict one or more to map one.
          Get a random one which is not in use */
@@ -1009,7 +1009,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
     }
 
-    if (unlikely(tid >= TESTCASE_ENTRIES)) {
+    if (unlikely(tid >= afl->q_testcase_max_cache_entries)) {
 
       // uh we were full, so now we have to search from start
       tid = afl->q_testcase_smallest_free;
@@ -1062,7 +1062,8 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q,
 
   if (unlikely(afl->q_testcase_cache_size + len >=
                    afl->q_testcase_max_cache_size ||
-               afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) {
+               afl->q_testcase_cache_count >=
+                   afl->q_testcase_max_cache_entries - 1)) {
 
     // no space? will be loaded regularly later.
     return;
-- 
cgit v1.2.3


From aa0d3785206d52c1815aff850817d55bf50f3598 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Fri, 23 Oct 2020 15:21:21 +0200
Subject: better cache entry algo

---
 src/afl-fuzz-queue.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index baa80e61..4989a0ba 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -985,6 +985,24 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
       /* Cache full. We neet to evict one or more to map one.
          Get a random one which is not in use */
 
+      if (unlikely(afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size &&
+          (afl->q_testcase_cache_count < afl->q_testcase_max_cache_entries &&
+           afl->q_testcase_max_cache_count <
+               afl->q_testcase_max_cache_entries))) {
+
+        if (afl->q_testcase_max_cache_count > afl->q_testcase_cache_count) {
+
+          afl->q_testcase_max_cache_entries =
+              afl->q_testcase_max_cache_count + 1;
+
+        } else {
+
+          afl->q_testcase_max_cache_entries = afl->q_testcase_cache_count + 1;
+
+        }
+
+      }
+
       do {
 
         // if the cache (MB) is not enough for the queue then this gets
-- 
cgit v1.2.3


From 4e99e3b36c8af6488fde10fac8cdb0797c95e02a Mon Sep 17 00:00:00 2001
From: Dominik Maier <domenukk@gmail.com>
Date: Sat, 24 Oct 2020 03:16:22 +0200
Subject: initializing testcase_buf

---
 src/afl-fuzz-queue.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 4989a0ba..02e66a4e 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -325,6 +325,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
   q->depth = afl->cur_depth + 1;
   q->passed_det = passed_det;
   q->trace_mini = NULL;
+  q->testcase_buf = NULL;
 
   if (q->depth > afl->max_depth) { afl->max_depth = q->depth; }
 
-- 
cgit v1.2.3


From e5f30c690822d69b4d99117024ae2570b1572481 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Sat, 24 Oct 2020 16:28:46 +0200
Subject: fix testcache bug

---
 src/afl-fuzz-queue.c | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 02e66a4e..d107dbc8 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -914,20 +914,22 @@ inline void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q,
 
   if (likely(q->testcase_buf)) {
 
+    u32 is_same = in == q->testcase_buf;
+
     if (likely(len != old_len)) {
 
-      afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len;
-      q->testcase_buf = realloc(q->testcase_buf, len);
+      u8 *ptr = realloc(q->testcase_buf, len);
 
-      if (unlikely(!q->testcase_buf)) {
+      if (likely(ptr)) {
 
-        PFATAL("Unable to malloc '%s' with len %d", q->fname, len);
+        q->testcase_buf = ptr;
+        afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len;
 
       }
 
     }
 
-    memcpy(q->testcase_buf, in, len);
+    if (unlikely(!is_same)) { memcpy(q->testcase_buf, in, len); }
 
   }
 
@@ -986,10 +988,12 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
       /* Cache full. We neet to evict one or more to map one.
          Get a random one which is not in use */
 
-      if (unlikely(afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size &&
-          (afl->q_testcase_cache_count < afl->q_testcase_max_cache_entries &&
-           afl->q_testcase_max_cache_count <
-               afl->q_testcase_max_cache_entries))) {
+      if (unlikely(afl->q_testcase_cache_size + len >=
+                       afl->q_testcase_max_cache_size &&
+                   (afl->q_testcase_cache_count <
+                        afl->q_testcase_max_cache_entries &&
+                    afl->q_testcase_max_cache_count <
+                        afl->q_testcase_max_cache_entries))) {
 
         if (afl->q_testcase_max_cache_count > afl->q_testcase_cache_count) {
 
-- 
cgit v1.2.3


From 868cb61ea6a2949e80e8a94fe7b19229bebecd10 Mon Sep 17 00:00:00 2001
From: van Hauser <vh@thc.org>
Date: Mon, 26 Oct 2020 15:24:33 +0100
Subject: hopeful the final testcache improvement ...

---
 src/afl-fuzz-queue.c | 56 +++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 47 insertions(+), 9 deletions(-)

(limited to 'src/afl-fuzz-queue.c')

diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index d107dbc8..c78df8be 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -979,21 +979,23 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
   if (unlikely(!q->testcase_buf)) {
 
     /* Buf not cached, let's load it */
-    u32 tid = afl->q_testcase_max_cache_count;
+    u32        tid = afl->q_testcase_max_cache_count;
+    static u32 do_once = 0;  // because even threaded we would want this. WIP
 
     while (unlikely(
         afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size ||
         afl->q_testcase_cache_count >= afl->q_testcase_max_cache_entries - 1)) {
 
-      /* Cache full. We neet to evict one or more to map one.
-         Get a random one which is not in use */
+      /* We want a max number of entries to the cache that we learn.
+         Very simple: once the cache is filled by size - that is the max. */
 
       if (unlikely(afl->q_testcase_cache_size + len >=
                        afl->q_testcase_max_cache_size &&
                    (afl->q_testcase_cache_count <
                         afl->q_testcase_max_cache_entries &&
                     afl->q_testcase_max_cache_count <
-                        afl->q_testcase_max_cache_entries))) {
+                        afl->q_testcase_max_cache_entries) &&
+                   !do_once)) {
 
         if (afl->q_testcase_max_cache_count > afl->q_testcase_cache_count) {
 
@@ -1006,8 +1008,19 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
         }
 
+        do_once = 1;
+        // release unneeded memory
+        u8 *ptr = ck_realloc(
+            afl->q_testcase_cache,
+            (afl->q_testcase_max_cache_entries + 1) * sizeof(size_t));
+
+        if (ptr) { afl->q_testcase_cache = (struct queue_entry **)ptr; }
+
       }
 
+      /* Cache full. We neet to evict one or more to map one.
+         Get a random one which is not in use */
+
       do {
 
         // if the cache (MB) is not enough for the queue then this gets
@@ -1065,11 +1078,16 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
     afl->q_testcase_cache[tid] = q;
     afl->q_testcase_cache_size += len;
     ++afl->q_testcase_cache_count;
-    if (tid >= afl->q_testcase_max_cache_count)
+    if (likely(tid >= afl->q_testcase_max_cache_count)) {
+
       afl->q_testcase_max_cache_count = tid + 1;
-    if (tid == afl->q_testcase_smallest_free)
+
+    } else if (unlikely(tid == afl->q_testcase_smallest_free)) {
+
       afl->q_testcase_smallest_free = tid + 1;
 
+    }
+
   }
 
   return q->testcase_buf;
@@ -1093,9 +1111,21 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q,
 
   }
 
-  u32 tid = 0;
+  u32 tid;
+
+  if (unlikely(afl->q_testcase_max_cache_count >=
+               afl->q_testcase_max_cache_entries)) {
 
-  while (likely(afl->q_testcase_cache[tid] != NULL))
+    // uh we were full, so now we have to search from start
+    tid = afl->q_testcase_smallest_free;
+
+  } else {
+
+    tid = afl->q_testcase_max_cache_count;
+
+  }
+
+  while (unlikely(afl->q_testcase_cache[tid] != NULL))
     ++tid;
 
   /* Map the test case into memory. */
@@ -1114,8 +1144,16 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q,
   afl->q_testcase_cache[tid] = q;
   afl->q_testcase_cache_size += len;
   ++afl->q_testcase_cache_count;
-  if (tid >= afl->q_testcase_max_cache_count)
+
+  if (likely(tid >= afl->q_testcase_max_cache_count)) {
+
     afl->q_testcase_max_cache_count = tid + 1;
 
+  } else if (unlikely(tid == afl->q_testcase_smallest_free)) {
+
+    afl->q_testcase_smallest_free = tid + 1;
+
+  }
+
 }
 
-- 
cgit v1.2.3