threadsafe doc fixes, code format

14 files changed, 106 insertions, 78 deletions

diff --git a/README.md b/README.md
index 69e2d14a..c04dba98 100644
--- a/README.md
+++ b/README.md
@@ -90,6 +90,7 @@ behaviours and defaults:
 
   | Feature/Instrumentation  | afl-gcc | llvm      | gcc_plugin | frida_mode | qemu_mode        |unicorn_mode |
   | -------------------------|:-------:|:---------:|:----------:|:----------:|:----------------:|:------------:|
+  | Threadsafe counters      |         |     x(3)  |            |            |                  |              |
   | NeverZero                | x86[_64]|     x(1)  |     x      |     x      |         x        |       x      |
   | Persistent Mode          |         |     x     |     x      |  x86[_64]  | x86[_64]/arm[64] |       x      |
   | LAF-Intel / CompCov      |         |     x     |            |            | x86[_64]/arm[64] | x86[_64]/arm |
@@ -104,7 +105,7 @@ behaviours and defaults:
 
   1. default for LLVM >= 9.0, env var for older version due an efficiency bug in previous llvm versions
   2. GCC creates non-performant code, hence it is disabled in gcc_plugin
-  3. (currently unassigned)
+  3. with `AFL_LLVM_THREADSAFE_INST`, disables NeverZero
   4. with pcguard mode and LTO mode for LLVM 11 and newer
   5. upcoming, development in the branch
   6. not compatible with LTO instrumentation and needs at least LLVM v4.1
diff --git a/docs/Changelog.md b/docs/Changelog.md
index d8ffe498..29ea918b 100644
--- a/docs/Changelog.md
+++ b/docs/Changelog.md
@@ -41,6 +41,8 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
       it fails
   - afl-cc:
     - We do not support llvm versions prior 6.0 anymore
+    - added thread safe counters to all modes (`AFL_LLVM_THREADSAFE_INST`),
+      note that this disables never zero counters.
     - Fix for -pie compiled binaries with default afl-clang-fast PCGUARD
     - Leak Sanitizer (AFL_USE_LSAN) added by Joshua Rogers, thanks!
     - Removed InsTrim instrumentation as it is not as good as PCGUARD
@@ -58,7 +60,6 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
     MacOS shared memory
   - updated the grammar custom mutator to the newest version
   - add -d (add dead fuzzer stats) to afl-whatsup
-  - add thread safe counters for LLVM CLASSIC (set AFL_LLVM_THREADSAFE_INST)
   - added AFL_PRINT_FILENAMES to afl-showmap/cmin to print the
     current filename
   - afl-showmap/cmin will now process queue items in alphabetical order
diff --git a/docs/env_variables.md b/docs/env_variables.md
index b4b866ab..38a67bc7 100644
--- a/docs/env_variables.md
+++ b/docs/env_variables.md
@@ -231,10 +231,11 @@ Then there are a few specific features that are only available in instrumentatio
 
   See [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md) for more information.
 
-### Thread safe instrumentation counters (in mode LLVM CLASSIC)
-   - Setting `AFL_LLVM_THREADSAFE_INST` will inject code that implements thread safe counters.
-     The overhead is a bit higher compared to the older non-thread safe case. 
-     `AFL_LLVM_NOT_ZERO` and `AFL_LLVM_SKIP_NEVERZERO` are supported (see below). 
+### Thread safe instrumentation counters (in all modes)
+
+   - Setting `AFL_LLVM_THREADSAFE_INST` will inject code that implements thread
+     safe counters. The overhead is a little bit higher compared to the older
+     non-thread safe case. Note that this disables neverzero (see below).
 
 ### NOT_ZERO
 
diff --git a/frida_mode/src/instrument/instrument_debug.c b/frida_mode/src/instrument/instrument_debug.c
index be72ef89..f8c1df77 100644
--- a/frida_mode/src/instrument/instrument_debug.c
+++ b/frida_mode/src/instrument/instrument_debug.c
@@ -17,7 +17,7 @@ static void instrument_debug(char *format, ...) {
   va_list ap;
   char    buffer[4096] = {0};
   int     ret;
-  int len;
+  int     len;
 
   va_start(ap, format);
   ret = vsnprintf(buffer, sizeof(buffer) - 1, format, ap);
diff --git a/frida_mode/src/stats/stats.c b/frida_mode/src/stats/stats.c
index 890a8d6b..662fb6d5 100644
--- a/frida_mode/src/stats/stats.c
+++ b/frida_mode/src/stats/stats.c
@@ -96,10 +96,10 @@ void stats_init(void) {
 void stats_vprint(int fd, char *format, va_list ap) {
 
   char buffer[4096] = {0};
-  int ret;
+  int  ret;
   int  len;
 
-  if(vsnprintf(buffer, sizeof(buffer) - 1, format, ap) < 0) { return; }
+  if (vsnprintf(buffer, sizeof(buffer) - 1, format, ap) < 0) { return; }
 
   len = strnlen(buffer, sizeof(buffer));
   IGNORED_RETURN(write(fd, buffer, len));
diff --git a/instrumentation/README.llvm.md b/instrumentation/README.llvm.md
index 02722588..8ce5afb9 100644
--- a/instrumentation/README.llvm.md
+++ b/instrumentation/README.llvm.md
@@ -144,9 +144,10 @@ is not optimal and was only fixed in llvm 9.
 You can set this with AFL_LLVM_NOT_ZERO=1
 See [README.neverzero.md](README.neverzero.md)
 
-Support for thread safe counters has been added for mode LLVM CLASSIC.
-Activate it with `AFL_LLVM_THREADSAFE_INST=1`. The tradeoff is better precision in 
-multi threaded apps for a slightly higher instrumentation overhead.
+Support for thread safe counters has been added for all modes.
+Activate it with `AFL_LLVM_THREADSAFE_INST=1`. The tradeoff is better precision
+in multi threaded apps for a slightly higher instrumentation overhead.
+This also disables the nozero counter default for performance reasons.
 
 ## 4) Snapshot feature
 
diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc
index 58969e18..20f1856e 100644
--- a/instrumentation/SanitizerCoverageLTO.so.cc
+++ b/instrumentation/SanitizerCoverageLTO.so.cc
@@ -1497,14 +1497,12 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
     }
 
     /* Update bitmap */
-    if (use_threadsafe_counters) { /* Atomic */
+    if (use_threadsafe_counters) {                                /* Atomic */
 
       IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One,
                           llvm::AtomicOrdering::Monotonic);
 
-    }
-    else
-    {
+    } else {
 
       LoadInst *Counter = IRB.CreateLoad(MapPtrIdx);
       Counter->setMetadata(Mo->getMDKindID("nosanitize"),
@@ -1524,6 +1522,7 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
           ->setMetadata(Mo->getMDKindID("nosanitize"), MDNode::get(*Ct, None));
 
     }
+
     // done :)
 
     inst++;
diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc
index dbddad0a..4a8c9e28 100644
--- a/instrumentation/SanitizerCoveragePCGUARD.so.cc
+++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc
@@ -1069,16 +1069,14 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
 
     /* Load counter for CurLoc */
 
-    Value *   MapPtrIdx = IRB.CreateGEP(MapPtr, CurLoc);
+    Value *MapPtrIdx = IRB.CreateGEP(MapPtr, CurLoc);
 
     if (use_threadsafe_counters) {
 
       IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One,
                           llvm::AtomicOrdering::Monotonic);
 
-    }
-    else
-    {
+    } else {
 
       LoadInst *Counter = IRB.CreateLoad(MapPtrIdx);
       /* Update bitmap */
diff --git a/instrumentation/afl-llvm-lto-instrumentation.so.cc b/instrumentation/afl-llvm-lto-instrumentation.so.cc
index b5fdb3d6..fe43fbe5 100644
--- a/instrumentation/afl-llvm-lto-instrumentation.so.cc
+++ b/instrumentation/afl-llvm-lto-instrumentation.so.cc
@@ -93,8 +93,8 @@ class AFLLTOPass : public ModulePass {
   uint32_t               function_minimum_size = 1;
   uint32_t               inst_blocks = 0, inst_funcs = 0, total_instr = 0;
   unsigned long long int map_addr = 0x10000;
-  const char *skip_nozero = NULL;
-  const char *use_threadsafe_counters = nullptr;
+  const char *           skip_nozero = NULL;
+  const char *           use_threadsafe_counters = nullptr;
 
 };
 
@@ -843,9 +843,12 @@ bool AFLLTOPass::runOnModule(Module &M) {
           /* Update bitmap */
 
           if (use_threadsafe_counters) {
+
             IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One,
                                 llvm::AtomicOrdering::Monotonic);
+
           } else {
+
             LoadInst *Counter = IRB.CreateLoad(MapPtrIdx);
             Counter->setMetadata(M.getMDKindID("nosanitize"),
                                  MDNode::get(C, None));
@@ -861,7 +864,9 @@ bool AFLLTOPass::runOnModule(Module &M) {
             }
 
             IRB.CreateStore(Incr, MapPtrIdx)
-                ->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None));
+                ->setMetadata(M.getMDKindID("nosanitize"),
+                              MDNode::get(C, None));
+
           }
 
           // done :)
diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc
index fe9e2e40..62f8b2ed 100644
--- a/instrumentation/afl-llvm-pass.so.cc
+++ b/instrumentation/afl-llvm-pass.so.cc
@@ -81,12 +81,12 @@ class AFLCoverage : public ModulePass {
   bool runOnModule(Module &M) override;
 
  protected:
-  uint32_t ngram_size = 0;
-  uint32_t ctx_k = 0;
-  uint32_t map_size = MAP_SIZE;
-  uint32_t function_minimum_size = 1;
-  const char *   ctx_str = NULL, *caller_str = NULL, *skip_nozero = NULL;
-  const char *   use_threadsafe_counters = nullptr;
+  uint32_t    ngram_size = 0;
+  uint32_t    ctx_k = 0;
+  uint32_t    map_size = MAP_SIZE;
+  uint32_t    function_minimum_size = 1;
+  const char *ctx_str = NULL, *caller_str = NULL, *skip_nozero = NULL;
+  const char *use_threadsafe_counters = nullptr;
 
 };
 
@@ -188,18 +188,30 @@ bool AFLCoverage::runOnModule(Module &M) {
   if ((isatty(2) && !getenv("AFL_QUIET")) || !!getenv("AFL_DEBUG")) {
 
     if (use_threadsafe_counters) {
-      if (!getenv("AFL_LLVM_NOT_ZERO")) {
-        skip_nozero = "1";
-        SAYF(cCYA "afl-llvm-pass" VERSION cRST " using thread safe counters\n");
-      }
-      else {
-        SAYF(cCYA "afl-llvm-pass" VERSION cRST
-                  " using thread safe not-zero-counters\n");
-      }
-    }
-    else
-    {
-      SAYF(cCYA "afl-llvm-pass" VERSION cRST " using non-thread safe instrumentation\n");
+
+      // disabled unless there is support for other modules as well
+      // (increases documentation complexity)
+      /*      if (!getenv("AFL_LLVM_NOT_ZERO")) { */
+
+      skip_nozero = "1";
+      SAYF(cCYA "afl-llvm-pass" VERSION cRST " using thread safe counters\n");
+
+      /*
+
+            } else {
+
+              SAYF(cCYA "afl-llvm-pass" VERSION cRST
+                        " using thread safe not-zero-counters\n");
+
+            }
+
+      */
+
+    } else {
+
+      SAYF(cCYA "afl-llvm-pass" VERSION cRST
+                " using non-thread safe instrumentation\n");
+
     }
 
   }
@@ -649,44 +661,44 @@ bool AFLCoverage::runOnModule(Module &M) {
 
       /* Update bitmap */
 
+      if (use_threadsafe_counters) {                              /* Atomic */
 
-      if (use_threadsafe_counters) {/* Atomic */
-
-  #if LLVM_VERSION_MAJOR < 9
+#if LLVM_VERSION_MAJOR < 9
         if (neverZero_counters_str !=
-            NULL) {  // with llvm 9 we make this the default as the bug in llvm is then fixed
-  #else
+            NULL) {  // with llvm 9 we make this the default as the bug in llvm
+                     // is then fixed
+#else
         if (!skip_nozero) {
 
-  #endif
+#endif
           // register MapPtrIdx in a todo list
           todo.push_back(MapPtrIdx);
 
-        }
-        else
-        {
+        } else {
+
           IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One,
                               llvm::AtomicOrdering::Monotonic);
+
         }
-      }
-      else
-      {
+
+      } else {
 
         LoadInst *Counter = IRB.CreateLoad(MapPtrIdx);
         Counter->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None));
 
         Value *Incr = IRB.CreateAdd(Counter, One);
 
-  #if LLVM_VERSION_MAJOR < 9
+#if LLVM_VERSION_MAJOR < 9
         if (neverZero_counters_str !=
-            NULL) {  // with llvm 9 we make this the default as the bug in llvm is
-                     // then fixed
-  #else
+            NULL) {  // with llvm 9 we make this the default as the bug in llvm
+                     // is then fixed
+#else
         if (!skip_nozero) {
 
-  #endif
+#endif
           /* hexcoder: Realize a counter that skips zero during overflow.
-           * Once this counter reaches its maximum value, it next increments to 1
+           * Once this counter reaches its maximum value, it next increments to
+           * 1
            *
            * Instead of
            * Counter + 1 -> Counter
@@ -705,7 +717,7 @@ bool AFLCoverage::runOnModule(Module &M) {
         IRB.CreateStore(Incr, MapPtrIdx)
             ->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None));
 
-      } /* non atomic case */
+      }                                                  /* non atomic case */
 
       /* Update prev_loc history vector (by placing cur_loc at the head of the
          vector and shuffle the other elements back by one) */
@@ -762,16 +774,19 @@ bool AFLCoverage::runOnModule(Module &M) {
 
     }
 
-    if (use_threadsafe_counters) { /*Atomic NeverZero */
+    if (use_threadsafe_counters) {                       /*Atomic NeverZero */
       // handle the list of registered blocks to instrument
       for (auto val : todo) {
-        /* hexcoder: Realize a thread-safe counter that skips zero during overflow. Once this counter reaches its maximum value, it next increments to 1
-             *
-             * Instead of
-             * Counter + 1 -> Counter
-             * we inject now this
-             * Counter + 1 -> {Counter, OverflowFlag}
-             * Counter + OverflowFlag -> Counter
+
+        /* hexcoder: Realize a thread-safe counter that skips zero during
+         * overflow. Once this counter reaches its maximum value, it next
+         * increments to 1
+         *
+         * Instead of
+         * Counter + 1 -> Counter
+         * we inject now this
+         * Counter + 1 -> {Counter, OverflowFlag}
+         * Counter + OverflowFlag -> Counter
          */
 
         /* equivalent c code looks like this
@@ -781,12 +796,19 @@ bool AFLCoverage::runOnModule(Module &M) {
             int old = atomic_load_explicit(&Counter, memory_order_relaxed);
             int new;
             do {
+
                  if (old == 255) {
+
                    new = 1;
+
                  } else {
+
                    new = old + 1;
+
                  }
+
             } while (!atomic_compare_exchange_weak_explicit(&Counter, &old, new,
+
          memory_order_relaxed, memory_order_relaxed));
 
          */
@@ -805,7 +827,8 @@ bool AFLCoverage::runOnModule(Module &M) {
 
         BasicBlock *BB = IRB.GetInsertBlock();
         // insert a basic block with the corpus of a do while loop
-        // the calculation may need to repeat, if atomic compare_exchange is not successful
+        // the calculation may need to repeat, if atomic compare_exchange is not
+        // successful
 
         BasicBlock::iterator it(*Counter);
         it++;  // split after load counter
@@ -857,6 +880,7 @@ bool AFLCoverage::runOnModule(Module &M) {
 
         // if the cmpXchg was not successful, retry
         IRB.CreateCondBr(Success, end_bb, do_while_bb);
+
       }
 
     }
diff --git a/qemu_mode/libqasan/libqasan.c b/qemu_mode/libqasan/libqasan.c
index d4742e3e..6ea24f08 100644
--- a/qemu_mode/libqasan/libqasan.c
+++ b/qemu_mode/libqasan/libqasan.c
@@ -69,9 +69,8 @@ __attribute__((constructor)) void __libqasan_init() {
   __libqasan_is_initialized = 1;
 
   __libqasan_init_hooks();
-  
-  if (getenv("AFL_INST_LIBS") || getenv("QASAN_HOTPACH"))
-    __libqasan_hotpatch();
+
+  if (getenv("AFL_INST_LIBS") || getenv("QASAN_HOTPACH")) __libqasan_hotpatch();
 
   if (getenv("AFL_INST_LIBS") || getenv("QASAN_HOTPACH")) __libqasan_hotpatch();
 
diff --git a/src/afl-cc.c b/src/afl-cc.c
index 6be6e165..486f7468 100644
--- a/src/afl-cc.c
+++ b/src/afl-cc.c
@@ -1777,7 +1777,8 @@ int main(int argc, char **argv, char **envp) {
         SAYF(
             "\nLLVM/LTO/afl-clang-fast/afl-clang-lto specific environment "
             "variables:\n"
-            "  AFL_LLVM_THREADSAFE_INST: instrument with thread safe counters\n"
+            "  AFL_LLVM_THREADSAFE_INST: instrument with thread safe counters, "
+            "disables neverzero\n"
 
             COUNTER_BEHAVIOUR
 
diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c
index 4a3e7f33..c3ce2edd 100644
--- a/src/afl-fuzz-one.c
+++ b/src/afl-fuzz-one.c
@@ -561,6 +561,7 @@ u8 fuzz_one_original(afl_state_t *afl) {
 
       if (afl->cmplog_lvl == 3 ||
           (afl->cmplog_lvl == 2 && afl->queue_cur->tc_ref) ||
+          afl->queue_cur->favored ||
           !(afl->fsrv.total_execs % afl->queued_paths) ||
           get_cur_time() - afl->last_path_time > 300000) {  // 300 seconds
 
diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c
index a3a623d9..5bdb4c8d 100644
--- a/src/afl-fuzz.c
+++ b/src/afl-fuzz.c
@@ -2066,13 +2066,10 @@ int main(int argc, char **argv_orig, char **envp) {
               break;
             case 4:
               afl->expand_havoc = 5;
-              if (afl->cmplog_lvl && afl->cmplog_lvl < 3) afl->cmplog_lvl = 3;
+              // if (afl->cmplog_lvl && afl->cmplog_lvl < 3) afl->cmplog_lvl =
+              // 3;
               break;
             case 5:
-              // if not in sync mode, enable deterministic mode?
-              // if (!afl->sync_id) afl->skip_deterministic = 0;
-              afl->expand_havoc = 6;
-            case 6:
               // nothing else currently
               break;