From f760e80729412a2cd44a12e76b81ccb433626e60 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 7 Sep 2021 17:15:54 +0200 Subject: add check_binary_signatures for afl-* utils --- src/afl-analyze.c | 1 + src/afl-common.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/afl-showmap.c | 2 ++ src/afl-tmin.c | 1 + 4 files changed, 68 insertions(+) (limited to 'src') diff --git a/src/afl-analyze.c b/src/afl-analyze.c index e19df3ce..eef08494 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -1093,6 +1093,7 @@ int main(int argc, char **argv_orig, char **envp) { parse_afl_kill_signal_env(getenv("AFL_KILL_SIGNAL"), SIGKILL); read_initial_file(); + (void)check_binary_signatures(fsrv.target_path); ACTF("Performing dry run (mem limit = %llu MB, timeout = %u ms%s)...", mem_limit, exec_tmout, edges_only ? ", edges only" : ""); diff --git a/src/afl-common.c b/src/afl-common.c index 9ca2b3e8..db19f0a7 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -25,8 +25,12 @@ #include #include +#define _GNU_SOURCE +#define __USE_GNU +#include #include #include +#include #include "debug.h" #include "alloc-inl.h" @@ -51,6 +55,66 @@ u8 last_intr = 0; #define AFL_PATH "/usr/local/lib/afl/" #endif +u32 check_binary_signatures(u8 *fn) { + + int ret = 0, fd = open(fn, O_RDONLY); + if (fd < 0) { PFATAL("Unable to open '%s'", fn); } + struct stat st; + if (fstat(fd, &st) < 0) { PFATAL("Unable to fstat '%s'", fn); } + u32 f_len = st.st_size; + u8 *f_data = mmap(0, f_len, PROT_READ, MAP_PRIVATE, fd, 0); + if (f_data == MAP_FAILED) { PFATAL("Unable to mmap file '%s'", fn); } + close(fd); + + if (memmem(f_data, f_len, PERSIST_SIG, strlen(PERSIST_SIG) + 1)) { + + if (!be_quiet) { OKF(cPIN "Persistent mode binary detected."); } + setenv(PERSIST_ENV_VAR, "1", 1); + ret = 1; + + } else if (getenv("AFL_PERSISTENT")) { + + if (!be_quiet) { + + WARNF("AFL_PERSISTENT is no longer supported and may misbehave!"); + + } + + } else if (getenv("AFL_FRIDA_PERSISTENT_ADDR")) { + + if (!be_quiet) { + + OKF("FRIDA Persistent mode configuration options detected."); + + } + + setenv(PERSIST_ENV_VAR, "1", 1); + ret = 1; + + } + + if (memmem(f_data, f_len, DEFER_SIG, strlen(DEFER_SIG) + 1)) { + + if (!be_quiet) { OKF(cPIN "Deferred forkserver binary detected."); } + setenv(DEFER_ENV_VAR, "1", 1); + ret += 2; + + } else if (getenv("AFL_DEFER_FORKSRV")) { + + if (!be_quiet) { + + WARNF("AFL_DEFER_FORKSRV is no longer supported and may misbehave!"); + + } + + } + + if (munmap(f_data, f_len)) { PFATAL("unmap() failed"); } + + return ret; + +} + void detect_file_args(char **argv, u8 *prog_in, bool *use_stdin) { u32 i = 0; diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 9122cd25..27b1e14a 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1189,6 +1189,8 @@ int main(int argc, char **argv_orig, char **envp) { } + (void)check_binary_signatures(fsrv->target_path); + shm_fuzz = ck_alloc(sizeof(sharedmem_t)); /* initialize cmplog_mode */ diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 792770e0..dff51e84 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -1209,6 +1209,7 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->shmem_fuzz = map + sizeof(u32); read_initial_file(); + (void)check_binary_signatures(fsrv->target_path); if (!fsrv->qemu_mode && !unicorn_mode) { -- cgit 1.4.1 From 8af84c203cfe241b5a8321c62387fd107ebf1031 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 7 Sep 2021 19:26:25 +0200 Subject: fix afl-showmap --- src/afl-showmap.c | 2 +- unicorn_mode/unicornafl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 27b1e14a..e143371e 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1189,7 +1189,7 @@ int main(int argc, char **argv_orig, char **envp) { } - (void)check_binary_signatures(fsrv->target_path); + if (in_dir) { (void)check_binary_signatures(fsrv->target_path); } shm_fuzz = ck_alloc(sizeof(sharedmem_t)); diff --git a/unicorn_mode/unicornafl b/unicorn_mode/unicornafl index 1c47d1eb..019b8715 160000 --- a/unicorn_mode/unicornafl +++ b/unicorn_mode/unicornafl @@ -1 +1 @@ -Subproject commit 1c47d1ebc7e904ad4efc1370f23e269fb9ac3f93 +Subproject commit 019b871539fe9ed3f41d882385a8b02c243d49ad -- cgit 1.4.1 From 4473904bc0de7011a77309d96f7090a51c8fe768 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 1 Oct 2021 13:25:02 +0200 Subject: fix -n --- docs/Changelog.md | 1 + src/afl-fuzz-stats.c | 5 +++-- src/afl-fuzz.c | 2 +- unicorn_mode/unicornafl | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/docs/Changelog.md b/docs/Changelog.md index de217c2e..dad5fee2 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -15,6 +15,7 @@ sending a mail to . information on how to deal with instrumenting libraries - fix a regression introduced in 3.10 that resulted in less coverage being detected. thanks to Collin May for reporting! + - fix -n dumb mode (nobody should use this) - afl-showmap, afl-tmin and afl-analyze now honor persistent mode for more speed. thanks to dloffre-snl for reporting! - afl-cc: diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index eb1fe2d9..870ba69a 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -560,8 +560,9 @@ void show_stats(afl_state_t *afl) { /* Roughly every minute, update fuzzer stats and save auto tokens. */ - if (unlikely(afl->force_ui_update || - cur_ms - afl->stats_last_stats_ms > STATS_UPDATE_SEC * 1000)) { + if (unlikely(!afl->non_instrumented_mode && + (afl->force_ui_update || + cur_ms - afl->stats_last_stats_ms > STATS_UPDATE_SEC * 1000))) { afl->stats_last_stats_ms = cur_ms; write_stats_file(afl, t_bytes, t_byte_ratio, stab_ratio, diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 8ffc0e77..87da9798 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1918,7 +1918,7 @@ int main(int argc, char **argv_orig, char **envp) { } - write_stats_file(afl, 0, 0, 0, 0); + if (!afl->non_instrumented_mode) { write_stats_file(afl, 0, 0, 0, 0); } maybe_update_plot_file(afl, 0, 0, 0); save_auto(afl); diff --git a/unicorn_mode/unicornafl b/unicorn_mode/unicornafl index 019b8715..c0e03d2c 160000 --- a/unicorn_mode/unicornafl +++ b/unicorn_mode/unicornafl @@ -1 +1 @@ -Subproject commit 019b871539fe9ed3f41d882385a8b02c243d49ad +Subproject commit c0e03d2c6b55a22025324f121746b41b1e756fb8 -- cgit 1.4.1 From e80131bef50d343e71a08cdf6ae1aa57b4475867 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 2 Oct 2021 22:47:22 +0200 Subject: fix some compiler warnings in 32-bit linux --- frida_mode/src/instrument/instrument_x64.c | 11 +++++++---- frida_mode/src/prefetch.c | 8 +++++--- include/config.h | 4 ++-- src/afl-analyze.c | 2 +- src/afl-fuzz.c | 2 +- src/afl-showmap.c | 2 +- src/afl-tmin.c | 2 +- unicorn_mode/unicornafl | 2 +- 8 files changed, 19 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/frida_mode/src/instrument/instrument_x64.c b/frida_mode/src/instrument/instrument_x64.c index 8948c4df..1c2cf113 100644 --- a/frida_mode/src/instrument/instrument_x64.c +++ b/frida_mode/src/instrument/instrument_x64.c @@ -144,7 +144,7 @@ static void instrument_coverate_write_function(GumStalkerOutput *output) { GumX86Writer *cw = output->writer.x86; GumAddress code_addr = 0; afl_log_code code = {0}; - guint64 instrument_hash_zero = 0; + /*guint64 instrument_hash_zero = 0;*/ if (current_log_impl == 0 || !gum_x86_writer_can_branch_directly_between(cw->pc, current_log_impl) || @@ -183,7 +183,8 @@ static void instrument_coverate_write_function(GumStalkerOutput *output) { } - *((gint *)&code.bytes[patch_offset1]) = (gint)current_pc_value1; + gint *dst_pc_value = (gint *)&code.bytes[patch_offset1]; + *dst_pc_value = (gint)current_pc_value1; gssize current_pc_value2 = GPOINTER_TO_SIZE(&instrument_previous_pc) - @@ -200,7 +201,8 @@ static void instrument_coverate_write_function(GumStalkerOutput *output) { } - *((gint *)&code.bytes[patch_offset2]) = (gint)current_pc_value2; + dst_pc_value = (gint *)&code.bytes[patch_offset2]; + *dst_pc_value = (gint)current_pc_value2; gsize afl_area_ptr_value = GPOINTER_TO_SIZE(__afl_area_ptr) - @@ -217,7 +219,8 @@ static void instrument_coverate_write_function(GumStalkerOutput *output) { } - *((gint *)&code.bytes[afl_area_ptr_offset]) = (gint)afl_area_ptr_value; + gint *dst_afl_area_ptr_value = (gint *)&code.bytes[afl_area_ptr_offset]; + *dst_afl_area_ptr_value = (gint)afl_area_ptr_value; gum_x86_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code)); diff --git a/frida_mode/src/prefetch.c b/frida_mode/src/prefetch.c index 0efbc9bf..c30ca65c 100644 --- a/frida_mode/src/prefetch.c +++ b/frida_mode/src/prefetch.c @@ -44,8 +44,9 @@ static void gum_afl_stalker_backpatcher_notify(GumStalkerObserver *self, sizeof(prefetch_data->backpatch_data) - prefetch_data->backpatch_size; if (sizeof(gsize) + size > remaining) { return; } - *(gsize *)(&prefetch_data->backpatch_data[prefetch_data->backpatch_size]) = - size; + gsize *dst_backpatch_size = (gsize *) + &prefetch_data->backpatch_data[prefetch_data->backpatch_size]; + *dst_backpatch_size = size; prefetch_data->backpatch_size += sizeof(gsize); memcpy(&prefetch_data->backpatch_data[prefetch_data->backpatch_size], @@ -115,7 +116,8 @@ static void prefetch_read_patches(void) { remaining > sizeof(gsize); remaining = prefetch_data->backpatch_size - offset) { - gsize size = *(gsize *)(&prefetch_data->backpatch_data[offset]); + gsize *src_backpatch_data = (gsize *)&prefetch_data->backpatch_data[offset]; + gsize size = *src_backpatch_data; offset += sizeof(gsize); if (prefetch_data->backpatch_size - offset < size) { diff --git a/include/config.h b/include/config.h index da74989e..4630da0c 100644 --- a/include/config.h +++ b/include/config.h @@ -237,11 +237,11 @@ (note that if this value is changed, several areas in afl-cc.c, afl-fuzz.c and afl-fuzz-state.c have to be changed as well! */ -#define MAX_FILE (1 * 1024 * 1024U) +#define MAX_FILE (1 * 1024 * 1024L) /* The same, for the test case minimizer: */ -#define TMIN_MAX_FILE (10 * 1024 * 1024) +#define TMIN_MAX_FILE (10 * 1024 * 1024L) /* Block normalization steps for afl-tmin: */ diff --git a/src/afl-analyze.c b/src/afl-analyze.c index eef08494..8295488d 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -184,7 +184,7 @@ static void read_initial_file(void) { if (st.st_size >= TMIN_MAX_FILE) { - FATAL("Input file is too large (%u MB max)", TMIN_MAX_FILE / 1024 / 1024); + FATAL("Input file is too large (%ld MB max)", TMIN_MAX_FILE / 1024 / 1024); } diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 87da9798..92a37697 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1348,7 +1348,7 @@ int main(int argc, char **argv_orig, char **envp) { } else if (afl->q_testcase_max_cache_size < 2 * MAX_FILE) { - FATAL("AFL_TESTCACHE_SIZE must be set to %u or more, or 0 to disable", + FATAL("AFL_TESTCACHE_SIZE must be set to %ld or more, or 0 to disable", (2 * MAX_FILE) % 1048576 == 0 ? (2 * MAX_FILE) / 1048576 : 1 + ((2 * MAX_FILE) / 1048576)); diff --git a/src/afl-showmap.c b/src/afl-showmap.c index e143371e..75b0ff99 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -413,7 +413,7 @@ static u32 read_file(u8 *in_file) { if (!be_quiet && !quiet_mode) { - WARNF("Input file '%s' is too large, only reading %u bytes.", in_file, + WARNF("Input file '%s' is too large, only reading %ld bytes.", in_file, MAX_FILE); } diff --git a/src/afl-tmin.c b/src/afl-tmin.c index dff51e84..4f3a6b80 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -221,7 +221,7 @@ static void read_initial_file(void) { if (st.st_size >= TMIN_MAX_FILE) { - FATAL("Input file is too large (%u MB max)", TMIN_MAX_FILE / 1024 / 1024); + FATAL("Input file is too large (%ld MB max)", TMIN_MAX_FILE / 1024 / 1024); } diff --git a/unicorn_mode/unicornafl b/unicorn_mode/unicornafl index c0e03d2c..1c47d1eb 160000 --- a/unicorn_mode/unicornafl +++ b/unicorn_mode/unicornafl @@ -1 +1 @@ -Subproject commit c0e03d2c6b55a22025324f121746b41b1e756fb8 +Subproject commit 1c47d1ebc7e904ad4efc1370f23e269fb9ac3f93 -- cgit 1.4.1 From a8844eaceb1df92635a327fc4edba082b102a2ff Mon Sep 17 00:00:00 2001 From: Kuang-che Wu Date: Mon, 11 Oct 2021 15:47:20 +0800 Subject: afl-showmap don't create empty "-" file --- src/afl-showmap.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 75b0ff99..a04c1f5b 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -242,9 +242,11 @@ static u32 write_results_to_file(afl_forkserver_t *fsrv, u8 *outfile) { if (cmin_mode && (fsrv->last_run_timed_out || (!caa && child_crashed != cco))) { - // create empty file to prevent error messages in afl-cmin - fd = open(outfile, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); - close(fd); + if (strcmp(outfile, "-")) { + // create empty file to prevent error messages in afl-cmin + fd = open(outfile, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); + close(fd); + } return ret; } -- cgit 1.4.1 From 00aa689f40a3c8276af257cf0b54dc655cb0423e Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 11 Oct 2021 14:28:17 +0200 Subject: fix accidental bystander kills --- docs/Changelog.md | 2 ++ qemu_mode/qemuafl | 2 +- src/afl-forkserver.c | 6 +++--- unicorn_mode/unicornafl | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/docs/Changelog.md b/docs/Changelog.md index dad5fee2..1c3830f9 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -18,6 +18,8 @@ sending a mail to . - fix -n dumb mode (nobody should use this) - afl-showmap, afl-tmin and afl-analyze now honor persistent mode for more speed. thanks to dloffre-snl for reporting! + - Prevent accidently killing non-afl/fuzz services when aborting + afl-showmap and other tools. - afl-cc: - fix for shared linking on MacOS - llvm and LTO mode verified to work with new llvm 14-dev diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl index 71ed0d20..a6758d1c 160000 --- a/qemu_mode/qemuafl +++ b/qemu_mode/qemuafl @@ -1 +1 @@ -Subproject commit 71ed0d206fd3d877420dceb4993a1011a4637ae6 +Subproject commit a6758d1cc3e4dde88fca3f0b3a903581b7c8b2e5 diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index c8c94c08..54f510c4 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -610,12 +610,12 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (!time_ms) { - kill(fsrv->fsrv_pid, fsrv->kill_signal); + if (fsrv->fsrv_pid > 0) { kill(fsrv->fsrv_pid, fsrv->kill_signal); } } else if (time_ms > fsrv->init_tmout) { fsrv->last_run_timed_out = 1; - kill(fsrv->fsrv_pid, fsrv->kill_signal); + if (fsrv->fsrv_pid > 0) { kill(fsrv->fsrv_pid, fsrv->kill_signal); } } else { @@ -1248,7 +1248,7 @@ fsrv_run_result_t afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, /* If there was no response from forkserver after timeout seconds, we kill the child. The forkserver should inform us afterwards */ - kill(fsrv->child_pid, fsrv->kill_signal); + if (fsrv->child_pid > 0) { kill(fsrv->child_pid, fsrv->kill_signal); } fsrv->last_run_timed_out = 1; if (read(fsrv->fsrv_st_fd, &fsrv->child_status, 4) < 4) { exec_ms = 0; } diff --git a/unicorn_mode/unicornafl b/unicorn_mode/unicornafl index f1c85364..c0e03d2c 160000 --- a/unicorn_mode/unicornafl +++ b/unicorn_mode/unicornafl @@ -1 +1 @@ -Subproject commit f1c853648a74b0157d233a2ef9f1693cfee78c11 +Subproject commit c0e03d2c6b55a22025324f121746b41b1e756fb8 -- cgit 1.4.1 From 8e662898095ed6ba283a87119e383948b83b8d75 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Tue, 12 Oct 2021 19:04:35 +0200 Subject: adapt compiler driver to laod new pass manager passes --- instrumentation/split-compares-pass.so.cc | 20 +++++++++++++------- src/afl-cc.c | 11 ++++++----- 2 files changed, 19 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index a0dbba7a..2ae6f893 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -1,6 +1,7 @@ /* * Copyright 2016 laf-intel * extended for floating point by Heiko Eißfeldt + * adapted to new pass manager by Heiko Eißfeldt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +34,7 @@ #include "llvm/Passes/PassBuilder.h" #include "llvm/IR/PassManager.h" //#include "llvm/IR/LegacyPassManager.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +//#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/IR/Module.h" @@ -61,7 +62,7 @@ namespace { class SplitComparesTransform : public PassInfoMixin { public: - static char ID; +// static char ID; SplitComparesTransform() : enableFPSplit(0) { initInstrumentList(); @@ -159,7 +160,7 @@ class SplitComparesTransform : public PassInfoMixin { extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK llvmGetPassPluginInfo() { return { - LLVM_PLUGIN_API_VERSION, "SplitCompares", "v0.1", + LLVM_PLUGIN_API_VERSION, "splitcompares", "v0.1", /* lambda to insert our pass into the pass pipeline. */ [](PassBuilder &PB) { #if 1 @@ -187,7 +188,7 @@ llvmGetPassPluginInfo() { }; } -char SplitComparesTransform::ID = 0; +//char SplitComparesTransform::ID = 0; /// This function splits FCMP instructions with xGE or xLE predicates into two /// FCMP instructions with predicate xGT or xLT and EQ @@ -700,7 +701,7 @@ bool SplitComparesTransform::splitCompare(CmpInst *cmp_inst, Module &M, ReplaceInstWithInst(cmp_inst->getParent()->getInstList(), ii, PN); // We split the comparison into low and high. If this isn't our target - // bitwidth we recursivly split the low and high parts again until we have + // bitwidth we recursively split the low and high parts again until we have // target bitwidth. if ((bitw / 2) > target_bitwidth) { @@ -1352,7 +1353,7 @@ PreservedAnalyses SplitComparesTransform::run(Module &M, ModuleAnalysisManager & if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) { - errs() << "Split-compare-pass by laf.intel@gmail.com, extended by " + errs() << "Split-compare-newpass by laf.intel@gmail.com, extended by " "heiko@hexco.de (splitting icmp to " << target_bitwidth << " bit)\n"; @@ -1364,7 +1365,7 @@ PreservedAnalyses SplitComparesTransform::run(Module &M, ModuleAnalysisManager & } - auto PA = PreservedAnalyses::all(); + auto PA = PreservedAnalyses::none(); if (enableFPSplit) { @@ -1447,6 +1448,11 @@ PreservedAnalyses SplitComparesTransform::run(Module &M, ModuleAnalysisManager & } + if ((isatty(2) && getenv("AFL_QUIET") == NULL) || + getenv("AFL_DEBUG") != NULL) { + errs() << count << " comparisons found\n"; + } + /* if (modified) { PA.abandon(); }*/ diff --git a/src/afl-cc.c b/src/afl-cc.c index e49addc4..f8621d72 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -500,11 +500,12 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; +// cc_params[cc_par_cnt++] = "-Xclang"; +// cc_params[cc_par_cnt++] = "-load"; +// cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = - alloc_printf("%s/split-compares-pass.so", obj_path); + alloc_printf("-fpass-plugin=%s/split-compares-pass.so", obj_path); } @@ -548,7 +549,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { #if LLVM_MAJOR >= 13 // fuck you llvm 13 - cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; +// cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; #endif if (lto_mode && !have_c) { -- cgit 1.4.1 From 544a65db5470359c18436eca123282d74fa47f2e Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Tue, 12 Oct 2021 23:02:15 +0200 Subject: converted afl-llvm-pass to new pass manager --- instrumentation/afl-llvm-pass.so.cc | 59 ++++++++++++++++++++++++++++++------- src/afl-cc.c | 10 ++----- 2 files changed, 51 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index ecf28f31..c2b87ecb 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -45,12 +45,15 @@ typedef long double max_align_t; #endif #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Passes/PassPlugin.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/IR/PassManager.h" +//#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +//#include "llvm/Transforms/IPO/PassManagerBuilder.h" #if LLVM_VERSION_MAJOR > 3 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) @@ -68,17 +71,18 @@ using namespace llvm; namespace { -class AFLCoverage : public ModulePass { +//class AFLCoverage : public ModulePass { +class AFLCoverage : public PassInfoMixin { public: - static char ID; - AFLCoverage() : ModulePass(ID) { +// static char ID; + AFLCoverage() { initInstrumentList(); } - bool runOnModule(Module &M) override; + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); protected: uint32_t ngram_size = 0; @@ -92,7 +96,38 @@ class AFLCoverage : public ModulePass { } // namespace -char AFLCoverage::ID = 0; +extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK +llvmGetPassPluginInfo() { + return { + LLVM_PLUGIN_API_VERSION, "AFLCoverage", "v0.1", + /* lambda to insert our pass into the pass pipeline. */ + [](PassBuilder &PB) { +#if 1 + using OptimizationLevel = typename PassBuilder::OptimizationLevel; + PB.registerOptimizerLastEPCallback( + [](ModulePassManager &MPM, OptimizationLevel OL) { + MPM.addPass(AFLCoverage()); + } + ); +/* TODO LTO registration */ +#else + using PipelineElement = typename PassBuilder::PipelineElement; + PB.registerPipelineParsingCallback( + [](StringRef Name, ModulePassManager &MPM, ArrayRef) { + if ( Name == "AFLCoverage" ) { + MPM.addPass(AFLCoverage); + return true; + } else { + return false; + } + } + ); +#endif + } + }; +} + +//char AFLCoverage::ID = 0; /* needed up to 3.9.0 */ #if LLVM_VERSION_MAJOR == 3 && \ @@ -118,7 +153,7 @@ uint64_t PowerOf2Ceil(unsigned in) { (LLVM_VERSION_MAJOR == 4 && LLVM_VERSION_PATCH >= 1) #define AFL_HAVE_VECTOR_INTRINSICS 1 #endif -bool AFLCoverage::runOnModule(Module &M) { +PreservedAnalyses AFLCoverage::run(Module &M, ModuleAnalysisManager &MAM) { LLVMContext &C = M.getContext(); @@ -133,6 +168,8 @@ bool AFLCoverage::runOnModule(Module &M) { u32 rand_seed; unsigned int cur_loc = 0; + auto PA = PreservedAnalyses::none(); + /* Setup random() so we get Actually Random(TM) outputs from AFL_R() */ gettimeofday(&tv, &tz); rand_seed = tv.tv_sec ^ tv.tv_usec ^ getpid(); @@ -969,10 +1006,10 @@ bool AFLCoverage::runOnModule(Module &M) { } - return true; + return PA; } - +#if 0 static void registerAFLPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -985,4 +1022,4 @@ static RegisterStandardPasses RegisterAFLPass( static RegisterStandardPasses RegisterAFLPass0( PassManagerBuilder::EP_EnabledOnOptLevel0, registerAFLPass); - +#endif diff --git a/src/afl-cc.c b/src/afl-cc.c index f8621d72..bbe548d9 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -500,12 +500,10 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -// cc_params[cc_par_cnt++] = "-Xclang"; -// cc_params[cc_par_cnt++] = "-load"; -// cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/split-compares-pass.so", obj_path); +// cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; } @@ -629,10 +627,8 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); + cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; + cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/afl-llvm-pass.so", obj_path); } -- cgit 1.4.1 From 6e08e809074763a9c4b35b65805e628689a2d562 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Tue, 12 Oct 2021 23:24:28 +0200 Subject: converted compare-transform-pass to new pass manager --- instrumentation/compare-transform-pass.so.cc | 69 ++++++++++++++++++++-------- src/afl-cc.c | 6 +-- 2 files changed, 52 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index 288e8282..e6695185 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -26,7 +26,10 @@ #include "llvm/ADT/Statistic.h" #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Passes/PassPlugin.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/IR/PassManager.h" +//#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -52,28 +55,16 @@ using namespace llvm; namespace { -class CompareTransform : public ModulePass { +class CompareTransform : public PassInfoMixin { public: - static char ID; - CompareTransform() : ModulePass(ID) { + CompareTransform() { initInstrumentList(); } - bool runOnModule(Module &M) override; - -#if LLVM_VERSION_MAJOR < 4 - const char *getPassName() const override { - -#else - StringRef getPassName() const override { - -#endif - return "transforms compare functions"; - - } + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); private: bool transformCmps(Module &M, const bool processStrcmp, @@ -85,7 +76,37 @@ class CompareTransform : public ModulePass { } // namespace -char CompareTransform::ID = 0; +extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK +llvmGetPassPluginInfo() { + return { + LLVM_PLUGIN_API_VERSION, "comparetransform", "v0.1", + /* lambda to insert our pass into the pass pipeline. */ + [](PassBuilder &PB) { +#if 1 + using OptimizationLevel = typename PassBuilder::OptimizationLevel; + PB.registerOptimizerLastEPCallback( + [](ModulePassManager &MPM, OptimizationLevel OL) { + MPM.addPass(CompareTransform()); + } + ); +/* TODO LTO registration */ +#else + using PipelineElement = typename PassBuilder::PipelineElement; + PB.registerPipelineParsingCallback( + [](StringRef Name, ModulePassManager &MPM, ArrayRef) { + if ( Name == "comparetransform" ) { + MPM.addPass(CompareTransform); + return true; + } else { + return false; + } + } + ); +#endif + } + }; +} + bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, const bool processMemcmp, @@ -592,7 +613,7 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, } -bool CompareTransform::runOnModule(Module &M) { +PreservedAnalyses CompareTransform::run(Module &M, ModuleAnalysisManager &MAM) { if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) printf( @@ -601,13 +622,22 @@ bool CompareTransform::runOnModule(Module &M) { else be_quiet = 1; + auto PA = PreservedAnalyses::none(); + transformCmps(M, true, true, true, true, true); verifyModule(M); - return true; +/* if (modified) { + PA.abandon(); + }*/ + + return PA; + +// return true; } +#if 0 static void registerCompTransPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -626,4 +656,5 @@ static RegisterStandardPasses RegisterCompTransPass0( static RegisterStandardPasses RegisterCompTransPassLTO( PassManagerBuilder::EP_FullLinkTimeOptimizationLast, registerCompTransPass); #endif +#endif diff --git a/src/afl-cc.c b/src/afl-cc.c index bbe548d9..a51632a2 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -480,11 +480,9 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = - alloc_printf("%s/compare-transform-pass.so", obj_path); + alloc_printf("-fpass-plugin=%s/compare-transform-pass.so", obj_path); } -- cgit 1.4.1 From 379c5806580dd58824df0f4fb7d215841d1bd459 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Tue, 12 Oct 2021 23:40:05 +0200 Subject: converted split-switches-pass to new pass manager --- instrumentation/split-compares-pass.so.cc | 2 +- instrumentation/split-switches-pass.so.cc | 71 +++++++++++++++++++++---------- src/afl-cc.c | 12 ++---- 3 files changed, 54 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index 2ae6f893..8d4935f5 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -176,7 +176,7 @@ llvmGetPassPluginInfo() { PB.registerPipelineParsingCallback( [](StringRef Name, ModulePassManager &MPM, ArrayRef) { if ( Name == "splitcompares" ) { - MPM.addPass(SplitComparesTransform); + MPM.addPass(SplitComparesTransform()); return true; } else { return false; diff --git a/instrumentation/split-switches-pass.so.cc b/instrumentation/split-switches-pass.so.cc index 82f198aa..ba143dca 100644 --- a/instrumentation/split-switches-pass.so.cc +++ b/instrumentation/split-switches-pass.so.cc @@ -27,11 +27,14 @@ #include "llvm/ADT/Statistic.h" #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Passes/PassPlugin.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/IR/PassManager.h" +//#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +//#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" @@ -54,28 +57,16 @@ using namespace llvm; namespace { -class SplitSwitchesTransform : public ModulePass { +class SplitSwitchesTransform : public PassInfoMixin { public: - static char ID; - SplitSwitchesTransform() : ModulePass(ID) { + SplitSwitchesTransform() { initInstrumentList(); } - bool runOnModule(Module &M) override; - -#if LLVM_VERSION_MAJOR >= 4 - StringRef getPassName() const override { - -#else - const char *getPassName() const override { - -#endif - return "splits switch constructs"; - - } + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); struct CaseExpr { @@ -103,7 +94,36 @@ class SplitSwitchesTransform : public ModulePass { } // namespace -char SplitSwitchesTransform::ID = 0; +extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK +llvmGetPassPluginInfo() { + return { + LLVM_PLUGIN_API_VERSION, "splitswitches", "v0.1", + /* lambda to insert our pass into the pass pipeline. */ + [](PassBuilder &PB) { +#if 1 + using OptimizationLevel = typename PassBuilder::OptimizationLevel; + PB.registerOptimizerLastEPCallback( + [](ModulePassManager &MPM, OptimizationLevel OL) { + MPM.addPass(SplitSwitchesTransform()); + } + ); +/* TODO LTO registration */ +#else + using PipelineElement = typename PassBuilder::PipelineElement; + PB.registerPipelineParsingCallback( + [](StringRef Name, ModulePassManager &MPM, ArrayRef) { + if ( Name == "splitswitches" ) { + MPM.addPass(SplitSwitchesTransform()); + return true; + } else { + return false; + } + } + ); +#endif + } + }; +} /* switchConvert - Transform simple list of Cases into list of CaseRange's */ BasicBlock *SplitSwitchesTransform::switchConvert( @@ -415,19 +435,26 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { } -bool SplitSwitchesTransform::runOnModule(Module &M) { +PreservedAnalyses SplitSwitchesTransform::run(Module &M, ModuleAnalysisManager &MAM) { if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) printf("Running split-switches-pass by laf.intel@gmail.com\n"); else be_quiet = 1; + + auto PA = PreservedAnalyses::none(); + splitSwitches(M); verifyModule(M); - return true; +/* if (modified) { + PA.abandon(); + }*/ -} + return PA; +} +#if 0 static void registerSplitSwitchesTransPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -447,4 +474,4 @@ static RegisterStandardPasses RegisterSplitSwitchesTransPassLTO( PassManagerBuilder::EP_FullLinkTimeOptimizationLast, registerSplitSwitchesTransPass); #endif - +#endif diff --git a/src/afl-cc.c b/src/afl-cc.c index a51632a2..e8584d50 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -460,11 +460,9 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = - alloc_printf("%s/split-switches-pass.so", obj_path); + alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); } @@ -531,11 +529,9 @@ static void edit_params(u32 argc, char **argv, char **envp) { alloc_printf("%s/cmplog-switches-pass.so", obj_path); // reuse split switches from laf - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = - alloc_printf("%s/split-switches-pass.so", obj_path); + alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); } -- cgit 1.4.1 From 3deca3b09b46130c9e23320c0b98f60543f9b5ba Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 15 Oct 2021 11:25:02 +0200 Subject: fix lto cmplog stability issue --- custom_mutators/grammar_mutator/grammar_mutator | 2 +- docs/Changelog.md | 1 + qemu_mode/qemuafl | 2 +- src/afl-fuzz-run.c | 30 +++++++++++++++++++++++-- unicorn_mode/unicornafl | 2 +- 5 files changed, 32 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/custom_mutators/grammar_mutator/grammar_mutator b/custom_mutators/grammar_mutator/grammar_mutator index eedf07dd..b79d51a8 160000 --- a/custom_mutators/grammar_mutator/grammar_mutator +++ b/custom_mutators/grammar_mutator/grammar_mutator @@ -1 +1 @@ -Subproject commit eedf07ddb0fb1f437f5e76b77cfd4064cf6a5d63 +Subproject commit b79d51a8daccbd7a693f9b6765c81ead14f28e26 diff --git a/docs/Changelog.md b/docs/Changelog.md index ea58a386..df4d343a 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -16,6 +16,7 @@ sending a mail to . - fix a regression introduced in 3.10 that resulted in less coverage being detected. thanks to Collin May for reporting! - fix -n dumb mode (nobody should use this) + - fix stability issue with LTO and cmplog - afl-showmap, afl-tmin and afl-analyze now honor persistent mode for more speed. thanks to dloffre-snl for reporting! - Prevent accidently killing non-afl/fuzz services when aborting diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl index a6758d1c..71ed0d20 160000 --- a/qemu_mode/qemuafl +++ b/qemu_mode/qemuafl @@ -1 +1 @@ -Subproject commit a6758d1cc3e4dde88fca3f0b3a903581b7c8b2e5 +Subproject commit 71ed0d206fd3d877420dceb4993a1011a4637ae6 diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 4173f4e1..da6ba7d9 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -291,8 +291,6 @@ static void write_with_gap(afl_state_t *afl, u8 *mem, u32 len, u32 skip_at, u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, u32 handicap, u8 from_queue) { - if (unlikely(afl->shm.cmplog_mode)) { q->exec_cksum = 0; } - u8 fault = 0, new_bits = 0, var_detected = 0, hnb = 0, first_run = (q->exec_cksum == 0); u64 start_us, stop_us, diff_us; @@ -300,6 +298,8 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, u32 use_tmout = afl->fsrv.exec_tmout; u8 *old_sn = afl->stage_name; + if (unlikely(afl->shm.cmplog_mode)) { q->exec_cksum = 0; } + /* Be a bit more generous about timeouts when resuming sessions, or when trying to calibrate already-added finds. This helps avoid trouble due to intermittent latency. */ @@ -343,6 +343,32 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, } + /* we need a dummy run if this is LTO + cmplog */ + if (unlikely(afl->shm.cmplog_mode)) { + + write_to_testcase(afl, use_mem, q->len); + + fault = fuzz_run_target(afl, &afl->fsrv, use_tmout); + + /* afl->stop_soon is set by the handler for Ctrl+C. When it's pressed, + we want to bail out quickly. */ + + if (afl->stop_soon || fault != afl->crash_mode) { goto abort_calibration; } + + if (!afl->non_instrumented_mode && !afl->stage_cur && + !count_bytes(afl, afl->fsrv.trace_bits)) { + + fault = FSRV_RUN_NOINST; + goto abort_calibration; + + } + +#ifdef INTROSPECTION + if (unlikely(!q->bitsmap_size)) q->bitsmap_size = afl->bitsmap_size; +#endif + + } + if (q->exec_cksum) { memcpy(afl->first_trace, afl->fsrv.trace_bits, afl->fsrv.map_size); diff --git a/unicorn_mode/unicornafl b/unicorn_mode/unicornafl index d4915053..f1c85364 160000 --- a/unicorn_mode/unicornafl +++ b/unicorn_mode/unicornafl @@ -1 +1 @@ -Subproject commit d4915053d477dd827b3fe4b494173d3fbf9f456e +Subproject commit f1c853648a74b0157d233a2ef9f1693cfee78c11 -- cgit 1.4.1 From 1f2fa22dad4440bf053e24811b5ece89ca276afc Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 16 Oct 2021 14:37:54 +0200 Subject: make new pass manager interface compiler version dependent (>=7) --- instrumentation/afl-llvm-pass.so.cc | 43 ++++++++++++++++++++----- instrumentation/compare-transform-pass.so.cc | 38 ++++++++++++++++++---- instrumentation/split-compares-pass.so.cc | 48 +++++++++++++++++++++++----- instrumentation/split-switches-pass.so.cc | 46 ++++++++++++++++++++++++-- src/afl-cc.c | 43 +++++++++++++++++++++++-- 5 files changed, 190 insertions(+), 28 deletions(-) (limited to 'src') diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index 92999443..75f8621b 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -45,15 +45,18 @@ typedef long double max_align_t; #endif #include "llvm/IR/IRBuilder.h" +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/IR/PassManager.h" -//#include "llvm/IR/LegacyPassManager.h" +#else +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" -//#include "llvm/Transforms/IPO/PassManagerBuilder.h" #if LLVM_VERSION_MAJOR > 3 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) @@ -71,18 +74,26 @@ using namespace llvm; namespace { -//class AFLCoverage : public ModulePass { +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ class AFLCoverage : public PassInfoMixin { - - public: -// static char ID; AFLCoverage() { + public: +#else +class AFLCoverage : public ModulePass { + public: + static char ID; + AFLCoverage() : ModulePass(ID) { +#endif initInstrumentList(); } +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +#else + bool runOnModule(Module &M) override; +#endif protected: uint32_t ngram_size = 0; @@ -96,6 +107,7 @@ class AFLCoverage : public PassInfoMixin { } // namespace +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK llvmGetPassPluginInfo() { return { @@ -126,8 +138,10 @@ llvmGetPassPluginInfo() { } }; } +#else -//char AFLCoverage::ID = 0; +char AFLCoverage::ID = 0; +#endif /* needed up to 3.9.0 */ #if LLVM_VERSION_MAJOR == 3 && \ @@ -153,7 +167,13 @@ uint64_t PowerOf2Ceil(unsigned in) { (LLVM_VERSION_MAJOR == 4 && LLVM_VERSION_PATCH >= 1) #define AFL_HAVE_VECTOR_INTRINSICS 1 #endif + + +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ PreservedAnalyses AFLCoverage::run(Module &M, ModuleAnalysisManager &MAM) { +#else +bool AFLCoverage::runOnModule(Module &M) { +#endif LLVMContext &C = M.getContext(); @@ -168,7 +188,9 @@ PreservedAnalyses AFLCoverage::run(Module &M, ModuleAnalysisManager &MAM) { u32 rand_seed; unsigned int cur_loc = 0; +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ auto PA = PreservedAnalyses::all(); +#endif /* Setup random() so we get Actually Random(TM) outputs from AFL_R() */ gettimeofday(&tv, &tz); @@ -1006,10 +1028,15 @@ PreservedAnalyses AFLCoverage::run(Module &M, ModuleAnalysisManager &MAM) { } +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ return PA; +#else + return true; +#endif } -#if 0 + +#if LLVM_VERSION_MAJOR < 7 /* use old pass manager */ static void registerAFLPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index ce8efaa7..3c975fe8 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -26,14 +26,17 @@ #include "llvm/ADT/Statistic.h" #include "llvm/IR/IRBuilder.h" +#if LLVM_MAJOR >= 7 /* use new pass manager */ #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/IR/PassManager.h" -//#include "llvm/IR/LegacyPassManager.h" +#else +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" @@ -55,16 +58,28 @@ using namespace llvm; namespace { +#if LLVM_MAJOR >= 7 /* use new pass manager */ class CompareTransform : public PassInfoMixin { public: CompareTransform() { +#else +class CompareTransform : public ModulePass { + + public: + static char ID; + CompareTransform() : ModulePass(ID) { +#endif initInstrumentList(); } +#if LLVM_MAJOR >= 7 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +#else + bool runOnModule(Module &M) override; +#endif private: bool transformCmps(Module &M, const bool processStrcmp, @@ -76,6 +91,7 @@ class CompareTransform : public PassInfoMixin { } // namespace +#if LLVM_MAJOR >= 7 /* use new pass manager */ extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK llvmGetPassPluginInfo() { return { @@ -106,7 +122,9 @@ llvmGetPassPluginInfo() { } }; } - +#else +char CompareTransform::ID = 0; +#endif bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, const bool processMemcmp, @@ -613,7 +631,11 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, } +#if LLVM_MAJOR >= 7 /* use new pass manager */ PreservedAnalyses CompareTransform::run(Module &M, ModuleAnalysisManager &MAM) { +#else +bool CompareTransform::runOnModule(Module &M) { +#endif if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) printf( @@ -622,22 +644,26 @@ PreservedAnalyses CompareTransform::run(Module &M, ModuleAnalysisManager &MAM) { else be_quiet = 1; +#if LLVM_MAJOR >= 7 /* use new pass manager */ auto PA = PreservedAnalyses::all(); +#endif transformCmps(M, true, true, true, true, true); verifyModule(M); +#if LLVM_MAJOR >= 7 /* use new pass manager */ /* if (modified) { PA.abandon(); }*/ return PA; - -// return true; +#else + return true; +#endif } -#if 0 +#if LLVM_MAJOR < 7 /* use old pass manager */ static void registerCompTransPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index 75a9c35c..ed7e111e 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -30,11 +30,14 @@ #include "llvm/Pass.h" #include "llvm/Support/raw_ostream.h" +#if LLVM_MAJOR >= 7 #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/IR/PassManager.h" -//#include "llvm/IR/LegacyPassManager.h" -//#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#else +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/IR/Module.h" @@ -58,17 +61,26 @@ using namespace llvm; namespace { -//class SplitComparesTransform : public ModulePass { +#if LLVM_MAJOR >= 7 class SplitComparesTransform : public PassInfoMixin { - public: // static char ID; SplitComparesTransform() : enableFPSplit(0) { +#else +class SplitComparesTransform : public ModulePass { + public: + static char ID; + SplitComparesTransform() : ModulePass(ID), enableFPSplit(0) { +#endif initInstrumentList(); } +#if LLVM_MAJOR >= 7 PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +#else + bool runOnModule(Module &M) override; +#endif private: int enableFPSplit; @@ -157,6 +169,7 @@ class SplitComparesTransform : public PassInfoMixin { } // namespace +#if LLVM_MAJOR >= 7 extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK llvmGetPassPluginInfo() { return { @@ -187,8 +200,9 @@ llvmGetPassPluginInfo() { } }; } - -//char SplitComparesTransform::ID = 0; +#else +char SplitComparesTransform::ID = 0; +#endif /// This function splits FCMP instructions with xGE or xLE predicates into two /// FCMP instructions with predicate xGT or xLT and EQ @@ -1342,7 +1356,11 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { } +#if LLVM_MAJOR >= 7 PreservedAnalyses SplitComparesTransform::run(Module &M, ModuleAnalysisManager &MAM) { +#else +bool SplitComparesTransform::runOnModule(Module &M) { +#endif char *bitw_env = getenv("AFL_LLVM_LAF_SPLIT_COMPARES_BITW"); if (!bitw_env) bitw_env = getenv("LAF_SPLIT_COMPARES_BITW"); @@ -1365,7 +1383,9 @@ PreservedAnalyses SplitComparesTransform::run(Module &M, ModuleAnalysisManager & } +#if LLVM_MAJOR >= 7 auto PA = PreservedAnalyses::all(); +#endif if (enableFPSplit) { @@ -1399,7 +1419,13 @@ PreservedAnalyses SplitComparesTransform::run(Module &M, ModuleAnalysisManager & auto op0 = CI->getOperand(0); auto op1 = CI->getOperand(1); - if (!op0 || !op1) { return PA; } + if (!op0 || !op1) { +#if LLVM_MAJOR >= 7 + return PA; +#else + return false; +#endif + } auto iTy1 = dyn_cast(op0->getType()); if (iTy1 && isa(op1->getType())) { @@ -1453,14 +1479,20 @@ PreservedAnalyses SplitComparesTransform::run(Module &M, ModuleAnalysisManager & errs() << count << " comparisons found\n"; } +#if LLVM_MAJOR >= 7 /* if (modified) { PA.abandon(); }*/ return PA; +#else + return true; +#endif } -#if 0 + +#if LLVM_MAJOR < 7 /* use old pass manager */ + static void registerSplitComparesPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { diff --git a/instrumentation/split-switches-pass.so.cc b/instrumentation/split-switches-pass.so.cc index b8cd61c3..42441de1 100644 --- a/instrumentation/split-switches-pass.so.cc +++ b/instrumentation/split-switches-pass.so.cc @@ -27,14 +27,17 @@ #include "llvm/ADT/Statistic.h" #include "llvm/IR/IRBuilder.h" +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/IR/PassManager.h" -//#include "llvm/IR/LegacyPassManager.h" +#else +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -//#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" @@ -57,16 +60,38 @@ using namespace llvm; namespace { +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ class SplitSwitchesTransform : public PassInfoMixin { public: SplitSwitchesTransform() { +#else +class SplitSwitchesTransform : public ModulePass { + public: + static char ID; + SplitSwitchesTransform() : ModulePass(ID) { +#endif initInstrumentList(); } +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +#else + bool runOnModule(Module &M) override; + +#if LLVM_VERSION_MAJOR >= 4 + StringRef getPassName() const override { + +#else + const char *getPassName() const override { + +#endif + return "splits switch constructs"; + + } +#endif struct CaseExpr { @@ -94,6 +119,7 @@ class SplitSwitchesTransform : public PassInfoMixin { } // namespace +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK llvmGetPassPluginInfo() { return { @@ -124,6 +150,9 @@ llvmGetPassPluginInfo() { } }; } +#else +char SplitSwitchesTransform::ID = 0; +#endif /* switchConvert - Transform simple list of Cases into list of CaseRange's */ BasicBlock *SplitSwitchesTransform::switchConvert( @@ -435,26 +464,37 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { } +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ PreservedAnalyses SplitSwitchesTransform::run(Module &M, ModuleAnalysisManager &MAM) { +#else +bool SplitSwitchesTransform::runOnModule(Module &M) { +#endif if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) printf("Running split-switches-pass by laf.intel@gmail.com\n"); else be_quiet = 1; +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ auto PA = PreservedAnalyses::all(); +#endif splitSwitches(M); verifyModule(M); +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ /* if (modified) { PA.abandon(); }*/ return PA; +#else + return true; +#endif } -#if 0 + +#if LLVM_VERSION_MAJOR < 7 /* use old pass manager */ static void registerSplitSwitchesTransPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { diff --git a/src/afl-cc.c b/src/afl-cc.c index e8584d50..7549e17b 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -460,10 +460,17 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); - +#else + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-load"; + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = + alloc_printf("%s/split-switches-pass.so", obj_path); +#endif } } @@ -478,9 +485,17 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { +#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/compare-transform-pass.so", obj_path); +#else + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-load"; + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = + alloc_printf("%s/compare-transform-pass.so", obj_path); +#endif } @@ -496,10 +511,18 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { +#if LLVM_MAJOR >= 7 cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/split-compares-pass.so", obj_path); // cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; +#else + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-load"; + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = + alloc_printf("%s/split-compares-pass.so", obj_path); +#endif } @@ -529,9 +552,17 @@ static void edit_params(u32 argc, char **argv, char **envp) { alloc_printf("%s/cmplog-switches-pass.so", obj_path); // reuse split switches from laf +#if LLVM_MAJOR >= 7 cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); +#else + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-load"; + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = + alloc_printf("%s/split-switches-pass.so", obj_path); +#endif } @@ -541,7 +572,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { #if LLVM_MAJOR >= 13 // fuck you llvm 13 -// cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; + cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; #endif if (lto_mode && !have_c) { @@ -621,9 +652,15 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { +#if LLVM_MAJOR >= 7 cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/afl-llvm-pass.so", obj_path); - +#else + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-load"; + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); +#endif } } -- cgit 1.4.1 From 8bc2b52f6579ab44f536d1ccb818acf37b047ec7 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 17 Oct 2021 13:03:01 +0200 Subject: format --- src/afl-showmap.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/afl-showmap.c b/src/afl-showmap.c index a04c1f5b..5df07bf2 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -243,10 +243,13 @@ static u32 write_results_to_file(afl_forkserver_t *fsrv, u8 *outfile) { (fsrv->last_run_timed_out || (!caa && child_crashed != cco))) { if (strcmp(outfile, "-")) { + // create empty file to prevent error messages in afl-cmin fd = open(outfile, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); close(fd); + } + return ret; } -- cgit 1.4.1 From ed10f3783bd8fab33ab5750f56bf87ed008f28ed Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 17 Oct 2021 13:05:33 +0200 Subject: new rtn cmplog: instrumentation side + supporting functions --- include/afl-fuzz.h | 1 + include/cmplog.h | 12 +- include/config.h | 4 +- instrumentation/afl-compiler-rt.o.c | 202 +++++++++++++++++++++++++++- instrumentation/cmplog-routines-pass.cc | 225 +++++++++++++++++++++++++++++++- src/afl-fuzz-one.c | 25 +++- src/afl-fuzz-queue.c | 91 ++++++++++++- src/afl-fuzz-stats.c | 3 +- src/afl-fuzz.c | 14 ++ 9 files changed, 551 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 4b19e698..5e52c0f0 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1136,6 +1136,7 @@ void setup_signal_handlers(void); void save_cmdline(afl_state_t *, u32, char **); void read_foreign_testcases(afl_state_t *, int); void write_crash_readme(afl_state_t *afl); +u8 check_if_text_buf(u8 *buf, u32 len); /* CmpLog */ diff --git a/include/cmplog.h b/include/cmplog.h index 878ed60c..88aa0a61 100644 --- a/include/cmplog.h +++ b/include/cmplog.h @@ -33,7 +33,7 @@ #define CMPLOG_LVL_MAX 3 #define CMP_MAP_W 65536 -#define CMP_MAP_H 32 +#define CMP_MAP_H 64 #define CMP_MAP_RTN_H (CMP_MAP_H / 4) #define SHAPE_BYTES(x) (x + 1) @@ -59,14 +59,16 @@ struct cmp_operands { u64 v0_128; u64 v1_128; -}; +} __attribute__((packed)); struct cmpfn_operands { - u8 v0[32]; - u8 v1[32]; + u8 v0[31]; + u8 v0_len; + u8 v1[31]; + u8 v1_len; -}; +} __attribute__((packed)); typedef struct cmp_operands cmp_map_list[CMP_MAP_H]; diff --git a/include/config.h b/include/config.h index 4630da0c..3b3b6daa 100644 --- a/include/config.h +++ b/include/config.h @@ -267,8 +267,8 @@ (first value), and to keep in memory as candidates. The latter should be much higher than the former. */ -#define USE_AUTO_EXTRAS 128 -#define MAX_AUTO_EXTRAS (USE_AUTO_EXTRAS * 64) +#define USE_AUTO_EXTRAS 4096 +#define MAX_AUTO_EXTRAS (USE_AUTO_EXTRAS * 8) /* Scaling factor for the effector map used to skip some of the more expensive deterministic steps. The actual divisor is set to diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 9acab4e7..21772ca0 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -1880,6 +1880,191 @@ static int area_is_valid(void *ptr, size_t len) { } +void __cmplog_rtn_hook_n(u8 *ptr1, u8 *ptr2, u32 len) { + + /* + u32 i; + if (area_is_valid(ptr1, 32) <= 0 || area_is_valid(ptr2, 32) <= 0) return; + fprintf(stderr, "rtn_n len=%u arg0=", len); + for (i = 0; i < len; i++) + fprintf(stderr, "%02x", ptr1[i]); + fprintf(stderr, " arg1="); + for (i = 0; i < len; i++) + fprintf(stderr, "%02x", ptr2[i]); + fprintf(stderr, "\n"); + */ + + if (likely(!__afl_cmp_map)) return; + // fprintf(stderr, "RTN1 %p %p %u\n", ptr1, ptr2, len); + if (unlikely(!len)) return; + int l = MIN(31, len); + + // fprintf(stderr, "RTN2 %u\n", l); + uintptr_t k = (uintptr_t)__builtin_return_address(0); + k = (k >> 4) ^ (k << 8); + k &= CMP_MAP_W - 1; + + u32 hits, reset = 1; + + if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].hits = 1; + __afl_cmp_map->headers[k].shape = l - 1; + reset = hits = 0; + + } else { + + hits = __afl_cmp_map->headers[k].hits++; + + if (__afl_cmp_map->headers[k].shape < l) { + + __afl_cmp_map->headers[k].shape = l - 1; + + } + + } + + struct cmpfn_operands *cmpfn = (struct cmpfn_operands *)__afl_cmp_map->log[k]; + hits &= CMP_MAP_RTN_H - 1; + if (unlikely(reset && !hits)) { + + __builtin_memset(cmpfn, 0, sizeof(struct cmpfn_operands)); + + } + + cmpfn[hits].v0_len = l; + cmpfn[hits].v1_len = l; + __builtin_memcpy(cmpfn[hits].v0, ptr1, l); + __builtin_memcpy(cmpfn[hits].v1, ptr2, l); + // fprintf(stderr, "RTN3\n"); + +} + +void __cmplog_rtn_hook_strn(u8 *ptr1, u8 *ptr2, u32 len) { + + /* + u32 i; + if (area_is_valid(ptr1, 32) <= 0 || area_is_valid(ptr2, 32) <= 0) return; + fprintf(stderr, "rtn_strn len=%u arg0=", len); + for (i = 0; i < len; i++) + fprintf(stderr, "%02x", ptr1[i]); + fprintf(stderr, " arg1="); + for (i = 0; i < len; i++) + fprintf(stderr, "%02x", ptr2[i]); + fprintf(stderr, "\n"); + */ + + if (likely(!__afl_cmp_map)) return; + // fprintf(stderr, "RTN1 %p %p %u\n", ptr1, ptr2, len); + if (unlikely(!len)) return; + int l = MIN(31, len + 1); + + // fprintf(stderr, "RTN2 %u\n", l); + uintptr_t k = (uintptr_t)__builtin_return_address(0); + k = (k >> 4) ^ (k << 8); + k &= CMP_MAP_W - 1; + + u32 hits, reset = 1; + + if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].hits = 1; + __afl_cmp_map->headers[k].shape = l - 1; + reset = hits = 0; + + } else { + + hits = __afl_cmp_map->headers[k].hits++; + + if (__afl_cmp_map->headers[k].shape < l) { + + __afl_cmp_map->headers[k].shape = l - 1; + + } + + } + + struct cmpfn_operands *cmpfn = (struct cmpfn_operands *)__afl_cmp_map->log[k]; + hits &= CMP_MAP_RTN_H - 1; + if (unlikely(reset && !hits)) { + + __builtin_memset(cmpfn, 0, sizeof(struct cmpfn_operands)); + + } + + cmpfn[hits].v0_len = 0x80 + l; + cmpfn[hits].v1_len = 0x80 + l; + __builtin_memcpy(cmpfn[hits].v0, ptr1, l); + __builtin_memcpy(cmpfn[hits].v1, ptr2, l); + // fprintf(stderr, "RTN3\n"); + +} + +void __cmplog_rtn_hook_str(u8 *ptr1, u8 *ptr2) { + + /* + u32 i; + if (area_is_valid(ptr1, 32) <= 0 || area_is_valid(ptr2, 32) <= 0) return; + fprintf(stderr, "rtn_str arg0="); + for (i = 0; i < len; i++) + fprintf(stderr, "%02x", ptr1[i]); + fprintf(stderr, " arg1="); + for (i = 0; i < len; i++) + fprintf(stderr, "%02x", ptr2[i]); + fprintf(stderr, "\n"); + */ + + if (likely(!__afl_cmp_map)) return; + // fprintf(stderr, "RTN1 %p %p\n", ptr1, ptr2); + if (unlikely(!ptr1 || !ptr2)) return; + int len1 = MIN(31, strlen(ptr1) + 1); + int len2 = MIN(31, strlen(ptr2) + 1); + int l = MIN(MAX(len1, len2), 31); + + // fprintf(stderr, "RTN2 %u\n", l); + uintptr_t k = (uintptr_t)__builtin_return_address(0); + k = (k >> 4) ^ (k << 8); + k &= CMP_MAP_W - 1; + + u32 hits, reset = 1; + + if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].hits = 1; + __afl_cmp_map->headers[k].shape = l - 1; + reset = hits = 0; + + } else { + + hits = __afl_cmp_map->headers[k].hits++; + + if (__afl_cmp_map->headers[k].shape < l) { + + __afl_cmp_map->headers[k].shape = l - 1; + + } + + } + + struct cmpfn_operands *cmpfn = (struct cmpfn_operands *)__afl_cmp_map->log[k]; + hits &= CMP_MAP_RTN_H - 1; + if (unlikely(reset && !hits)) { + + __builtin_memset(cmpfn, 0, sizeof(struct cmpfn_operands)); + + } + + cmpfn[hits].v0_len = 0x80 + len1; + cmpfn[hits].v1_len = 0x80 + len2; + __builtin_memcpy(cmpfn[hits].v0, ptr1, len1); + __builtin_memcpy(cmpfn[hits].v1, ptr2, len2); + // fprintf(stderr, "RTN3\n"); + +} + void __cmplog_rtn_hook(u8 *ptr1, u8 *ptr2) { /* @@ -1907,14 +2092,14 @@ void __cmplog_rtn_hook(u8 *ptr1, u8 *ptr2) { k = (k >> 4) ^ (k << 8); k &= CMP_MAP_W - 1; - u32 hits; + u32 hits, reset = 1; if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; __afl_cmp_map->headers[k].hits = 1; __afl_cmp_map->headers[k].shape = len - 1; - hits = 0; + reset = hits = 0; } else { @@ -1928,11 +2113,16 @@ void __cmplog_rtn_hook(u8 *ptr1, u8 *ptr2) { } + struct cmpfn_operands *cmpfn = (struct cmpfn_operands *)__afl_cmp_map->log[k]; hits &= CMP_MAP_RTN_H - 1; - __builtin_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0, - ptr1, len); - __builtin_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1, - ptr2, len); + if (unlikely(reset && !hits)) { + + __builtin_memset(cmpfn, 0, sizeof(struct cmpfn_operands)); + + } + + __builtin_memcpy(cmpfn[hits].v0, ptr1, len); + __builtin_memcpy(cmpfn[hits].v1, ptr2, len); // fprintf(stderr, "RTN3\n"); } diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index 1e2610f2..56f1a083 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -87,12 +87,14 @@ char CmpLogRoutines::ID = 0; bool CmpLogRoutines::hookRtns(Module &M) { - std::vector calls, llvmStdStd, llvmStdC, gccStdStd, gccStdC; - LLVMContext & C = M.getContext(); + std::vector calls, llvmStdStd, llvmStdC, gccStdStd, gccStdC, + Memcmp, Strcmp, Strncmp; + LLVMContext &C = M.getContext(); Type *VoidTy = Type::getVoidTy(C); // PointerType *VoidPtrTy = PointerType::get(VoidTy, 0); IntegerType *Int8Ty = IntegerType::getInt8Ty(C); + IntegerType *Int32Ty = IntegerType::getInt32Ty(C); PointerType *i8PtrTy = PointerType::get(Int8Ty, 0); #if LLVM_VERSION_MAJOR < 9 @@ -184,6 +186,60 @@ bool CmpLogRoutines::hookRtns(Module &M) { FunctionCallee cmplogGccStdC = c4; #endif +#if LLVM_VERSION_MAJOR < 9 + Constant * +#else + FunctionCallee +#endif + c5 = M.getOrInsertFunction("__cmplog_rtn_hook_n", VoidTy, i8PtrTy, + i8PtrTy, Int32Ty +#if LLVM_VERSION_MAJOR < 5 + , + NULL +#endif + ); +#if LLVM_VERSION_MAJOR < 9 + Function *cmplogHookFnN = cast(c5); +#else + FunctionCallee cmplogHookFnN = c5; +#endif + +#if LLVM_VERSION_MAJOR < 9 + Constant * +#else + FunctionCallee +#endif + c6 = M.getOrInsertFunction("__cmplog_rtn_hook_strn", VoidTy, i8PtrTy, + i8PtrTy, Int32Ty +#if LLVM_VERSION_MAJOR < 5 + , + NULL +#endif + ); +#if LLVM_VERSION_MAJOR < 9 + Function *cmplogHookFnStrN = cast(c6); +#else + FunctionCallee cmplogHookFnStrN = c6; +#endif + +#if LLVM_VERSION_MAJOR < 9 + Constant * +#else + FunctionCallee +#endif + c7 = M.getOrInsertFunction("__cmplog_rtn_hook_str", VoidTy, i8PtrTy, + i8PtrTy +#if LLVM_VERSION_MAJOR < 5 + , + NULL +#endif + ); +#if LLVM_VERSION_MAJOR < 9 + Function *cmplogHookFnStr = cast(c7); +#else + FunctionCallee cmplogHookFnStr = c7; +#endif + GlobalVariable *AFLCmplogPtr = M.getNamedGlobal("__afl_cmp_map"); if (!AFLCmplogPtr) { @@ -214,12 +270,77 @@ bool CmpLogRoutines::hookRtns(Module &M) { if (callInst->getCallingConv() != llvm::CallingConv::C) continue; FunctionType *FT = Callee->getFunctionType(); + std::string FuncName = Callee->getName().str(); bool isPtrRtn = FT->getNumParams() >= 2 && !FT->getReturnType()->isVoidTy() && FT->getParamType(0) == FT->getParamType(1) && FT->getParamType(0)->isPointerTy(); + bool isPtrRtnN = FT->getNumParams() >= 3 && + !FT->getReturnType()->isVoidTy() && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0)->isPointerTy() && + FT->getParamType(2)->isIntegerTy(); + + bool isMemcmp = + (!FuncName.compare("memcmp") || !FuncName.compare("bcmp") || + !FuncName.compare("CRYPTO_memcmp") || + !FuncName.compare("OPENSSL_memcmp") || + !FuncName.compare("memcmp_const_time") || + !FuncName.compare("memcmpct")); + isMemcmp &= FT->getNumParams() == 3 && + FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0)->isPointerTy() && + FT->getParamType(1)->isPointerTy() && + FT->getParamType(2)->isIntegerTy(); + + bool isStrcmp = + (!FuncName.compare("strcmp") || !FuncName.compare("xmlStrcmp") || + !FuncName.compare("xmlStrEqual") || + !FuncName.compare("g_strcmp0") || + !FuncName.compare("curl_strequal") || + !FuncName.compare("strcsequal") || + !FuncName.compare("strcasecmp") || + !FuncName.compare("stricmp") || + !FuncName.compare("ap_cstr_casecmp") || + !FuncName.compare("OPENSSL_strcasecmp") || + !FuncName.compare("xmlStrcasecmp") || + !FuncName.compare("g_strcasecmp") || + !FuncName.compare("g_ascii_strcasecmp") || + !FuncName.compare("Curl_strcasecompare") || + !FuncName.compare("Curl_safe_strcasecompare") || + !FuncName.compare("cmsstrcasecmp") || + !FuncName.compare("strstr") || + !FuncName.compare("g_strstr_len") || + !FuncName.compare("ap_strcasestr") || + !FuncName.compare("xmlStrstr") || + !FuncName.compare("xmlStrcasestr") || + !FuncName.compare("g_str_has_prefix") || + !FuncName.compare("g_str_has_suffix")); + isStrcmp &= + FT->getNumParams() == 2 && FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == IntegerType::getInt8PtrTy(M.getContext()); + + bool isStrncmp = (!FuncName.compare("strncmp") || + !FuncName.compare("xmlStrncmp") || + !FuncName.compare("curl_strnequal") || + !FuncName.compare("strncasecmp") || + !FuncName.compare("strnicmp") || + !FuncName.compare("ap_cstr_casecmpn") || + !FuncName.compare("OPENSSL_strncasecmp") || + !FuncName.compare("xmlStrncasecmp") || + !FuncName.compare("g_ascii_strncasecmp") || + !FuncName.compare("Curl_strncasecompare") || + !FuncName.compare("g_strncasecmp")); + isStrncmp &= FT->getNumParams() == 3 && + FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8PtrTy(M.getContext()) && + FT->getParamType(2)->isIntegerTy(); + bool isGccStdStringStdString = Callee->getName().find("__is_charIT_EE7__value") != std::string::npos && @@ -267,13 +388,19 @@ bool CmpLogRoutines::hookRtns(Module &M) { */ if (isGccStdStringCString || isGccStdStringStdString || - isLlvmStdStringStdString || isLlvmStdStringCString) { + isLlvmStdStringStdString || isLlvmStdStringCString || isMemcmp || + isStrcmp || isStrncmp) { - isPtrRtn = false; + isPtrRtnN = isPtrRtn = false; } + if (isPtrRtnN) { isPtrRtn = false; } + if (isPtrRtn) { calls.push_back(callInst); } + if (isMemcmp || isPtrRtnN) { Memcmp.push_back(callInst); } + if (isStrcmp) { Strcmp.push_back(callInst); } + if (isStrncmp) { Strncmp.push_back(callInst); } if (isGccStdStringStdString) { gccStdStd.push_back(callInst); } if (isGccStdStringCString) { gccStdC.push_back(callInst); } if (isLlvmStdStringStdString) { llvmStdStd.push_back(callInst); } @@ -288,7 +415,8 @@ bool CmpLogRoutines::hookRtns(Module &M) { } if (!calls.size() && !gccStdStd.size() && !gccStdC.size() && - !llvmStdStd.size() && !llvmStdC.size()) + !llvmStdStd.size() && !llvmStdC.size() && !Memcmp.size() && + Strcmp.size() && Strncmp.size()) return false; /* @@ -323,6 +451,93 @@ bool CmpLogRoutines::hookRtns(Module &M) { } + for (auto &callInst : Memcmp) { + + Value *v1P = callInst->getArgOperand(0), *v2P = callInst->getArgOperand(1), + *v3P = callInst->getArgOperand(2); + + IRBuilder<> IRB2(callInst->getParent()); + IRB2.SetInsertPoint(callInst); + + LoadInst *CmpPtr = IRB2.CreateLoad(AFLCmplogPtr); + CmpPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + auto is_not_null = IRB2.CreateICmpNE(CmpPtr, Null); + auto ThenTerm = SplitBlockAndInsertIfThen(is_not_null, callInst, false); + + IRBuilder<> IRB(ThenTerm); + + std::vector args; + Value * v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); + Value * v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); + Value * v3Pcasted = IRB.CreateZExtOrBitCast(v3P, Int32Ty); + args.push_back(v1Pcasted); + args.push_back(v2Pcasted); + args.push_back(v3Pcasted); + + IRB.CreateCall(cmplogHookFnN, args); + + // errs() << callInst->getCalledFunction()->getName() << "\n"; + + } + + for (auto &callInst : Strcmp) { + + Value *v1P = callInst->getArgOperand(0), *v2P = callInst->getArgOperand(1), + *v3P = callInst->getArgOperand(2); + + IRBuilder<> IRB2(callInst->getParent()); + IRB2.SetInsertPoint(callInst); + + LoadInst *CmpPtr = IRB2.CreateLoad(AFLCmplogPtr); + CmpPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + auto is_not_null = IRB2.CreateICmpNE(CmpPtr, Null); + auto ThenTerm = SplitBlockAndInsertIfThen(is_not_null, callInst, false); + + IRBuilder<> IRB(ThenTerm); + + std::vector args; + Value * v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); + Value * v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); + Value * v3Pcasted = IRB.CreateZExtOrBitCast(v3P, Int32Ty); + args.push_back(v1Pcasted); + args.push_back(v2Pcasted); + args.push_back(v3Pcasted); + + IRB.CreateCall(cmplogHookFnStr, args); + + // errs() << callInst->getCalledFunction()->getName() << "\n"; + + } + + for (auto &callInst : Strncmp) { + + Value *v1P = callInst->getArgOperand(0), *v2P = callInst->getArgOperand(1), + *v3P = callInst->getArgOperand(2); + + IRBuilder<> IRB2(callInst->getParent()); + IRB2.SetInsertPoint(callInst); + + LoadInst *CmpPtr = IRB2.CreateLoad(AFLCmplogPtr); + CmpPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + auto is_not_null = IRB2.CreateICmpNE(CmpPtr, Null); + auto ThenTerm = SplitBlockAndInsertIfThen(is_not_null, callInst, false); + + IRBuilder<> IRB(ThenTerm); + + std::vector args; + Value * v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); + Value * v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); + Value * v3Pcasted = IRB.CreateZExtOrBitCast(v3P, Int32Ty); + args.push_back(v1Pcasted); + args.push_back(v2Pcasted); + args.push_back(v3Pcasted); + + IRB.CreateCall(cmplogHookFnStrN, args); + + // errs() << callInst->getCalledFunction()->getName() << "\n"; + + } + for (auto &callInst : gccStdStd) { Value *v1P = callInst->getArgOperand(0), *v2P = callInst->getArgOperand(1); diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 17749601..3217fb0f 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -448,11 +448,11 @@ u8 fuzz_one_original(afl_state_t *afl) { ACTF( "Fuzzing test case #%u (%u total, %llu uniq crashes found, " - "perf_score=%0.0f, exec_us=%llu, hits=%u, map=%u)...", + "perf_score=%0.0f, exec_us=%llu, hits=%u, map=%u, ascii=%u)...", afl->current_entry, afl->queued_paths, afl->unique_crashes, afl->queue_cur->perf_score, afl->queue_cur->exec_us, likely(afl->n_fuzz) ? afl->n_fuzz[afl->queue_cur->n_fuzz_entry] : 0, - afl->queue_cur->bitmap_size); + afl->queue_cur->bitmap_size, afl->queue_cur->is_ascii); fflush(stdout); } @@ -2003,11 +2003,16 @@ havoc_stage: where we take the input file and make random stacked tweaks. */ #define MAX_HAVOC_ENTRY 59 /* 55 to 60 */ +#define MUTATE_ASCII_DICT 64 u32 r_max, r; r_max = (MAX_HAVOC_ENTRY + 1) + (afl->extras_cnt ? 4 : 0) + - (afl->a_extras_cnt ? 4 : 0); + (afl->a_extras_cnt + ? (unlikely(afl->cmplog_binary && afl->queue_cur->is_ascii) + ? MUTATE_ASCII_DICT + : 4) + : 0); if (unlikely(afl->expand_havoc && afl->ready_for_splicing_count > 1)) { @@ -2592,7 +2597,15 @@ havoc_stage: if (afl->a_extras_cnt) { - if (r < 2) { + u32 r_cmp = 2; + + if (unlikely(afl->cmplog_binary && afl->queue_cur->is_ascii)) { + + r_cmp = MUTATE_ASCII_DICT >> 1; + + } + + if (r < r_cmp) { /* Use the dictionary. */ @@ -2612,7 +2625,7 @@ havoc_stage: break; - } else if (r < 4) { + } else if (r < (r_cmp << 1)) { u32 use_extra = rand_below(afl, afl->a_extras_cnt); u32 extra_len = afl->a_extras[use_extra].len; @@ -2641,7 +2654,7 @@ havoc_stage: } else { - r -= 4; + r -= (r_cmp << 1); } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 16af2c6b..718f7cb6 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -315,7 +315,96 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) { } -/* check if ascii or UTF-8 */ +/* check if pointer is ascii or UTF-8 */ + +u8 check_if_text_buf(u8 *buf, u32 len) { + + u32 offset = 0, ascii = 0, utf8 = 0; + + while (offset < len) { + + // ASCII: <= 0x7F to allow ASCII control characters + if ((buf[offset + 0] == 0x09 || buf[offset + 0] == 0x0A || + buf[offset + 0] == 0x0D || + (0x20 <= buf[offset + 0] && buf[offset + 0] <= 0x7E))) { + + offset++; + utf8++; + ascii++; + continue; + + } + + if (isascii((int)buf[offset]) || isprint((int)buf[offset])) { + + ascii++; + // we continue though as it can also be a valid utf8 + + } + + // non-overlong 2-byte + if (len - offset > 1 && + ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF))) { + + offset += 2; + utf8++; + continue; + + } + + // excluding overlongs + if ((len - offset > 2) && + ((buf[offset + 0] == 0xE0 && + (0xA0 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && + buf[offset + 2] <= 0xBF)) || // straight 3-byte + (((0xE1 <= buf[offset + 0] && buf[offset + 0] <= 0xEC) || + buf[offset + 0] == 0xEE || buf[offset + 0] == 0xEF) && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && + buf[offset + 2] <= 0xBF)) || // excluding surrogates + (buf[offset + 0] == 0xED && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0x9F) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF)))) { + + offset += 3; + utf8++; + continue; + + } + + // planes 1-3 + if ((len - offset > 3) && + ((buf[offset + 0] == 0xF0 && + (0x90 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) && + (0x80 <= buf[offset + 3] && + buf[offset + 3] <= 0xBF)) || // planes 4-15 + ((0xF1 <= buf[offset + 0] && buf[offset + 0] <= 0xF3) && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) && + (0x80 <= buf[offset + 3] && buf[offset + 3] <= 0xBF)) || // plane 16 + (buf[offset + 0] == 0xF4 && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0x8F) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) && + (0x80 <= buf[offset + 3] && buf[offset + 3] <= 0xBF)))) { + + offset += 4; + utf8++; + continue; + + } + + offset++; + + } + + return (utf8 > ascii ? utf8 : ascii); + +} + +/* check if queue entry is ascii or UTF-8 */ static u8 check_if_text(afl_state_t *afl, struct queue_entry *q) { diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 870ba69a..7796036b 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -278,6 +278,7 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, "total_edges : %u\n" "var_byte_count : %u\n" "havoc_expansion : %u\n" + "auto_dict_entries : %u\n" "testcache_size : %llu\n" "testcache_count : %u\n" "testcache_evict : %u\n" @@ -316,7 +317,7 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, -1, #endif t_bytes, afl->fsrv.real_map_size, afl->var_byte_count, - afl->expand_havoc, afl->q_testcase_cache_size, + afl->expand_havoc, afl->a_extras_cnt, afl->q_testcase_cache_size, afl->q_testcase_cache_count, afl->q_testcase_evictions, afl->use_banner, afl->unicorn_mode ? "unicorn" : "", afl->fsrv.qemu_mode ? "qemu " : "", diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 92a37697..6ec033b2 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2261,6 +2261,20 @@ stop_fuzzing: } + if (afl->not_on_tty) { + + u32 t_bytes = count_non_255_bytes(afl, afl->virgin_bits); + u8 time_tmp[64]; + u_stringify_time_diff(time_tmp, get_cur_time(), afl->start_time); + ACTF( + "Statistics: %u new paths found, %.02f%% coverage achieved, %llu " + "crashes found, %llu timeouts found, total runtime %s", + afl->queued_discovered, + ((double)t_bytes * 100) / afl->fsrv.real_map_size, afl->unique_crashes, + afl->unique_hangs, time_tmp); + + } + #ifdef PROFILING SAYF(cYEL "[!] " cRST "Profiling information: %llu ms total work, %llu ns/run\n", -- cgit 1.4.1 From 2363a047500c133996e40808f1974e4aa97ed5bf Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 17 Oct 2021 14:21:08 +0200 Subject: simplify SHAPE_BYTES(h->shape) --- src/afl-fuzz-redqueen.c | 97 ++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 46 deletions(-) (limited to 'src') diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 268f726c..2d610132 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -64,6 +64,8 @@ struct range { }; +static u32 hshape; + static struct range *add_range(struct range *ranges, u32 start, u32 end) { struct range *r = ck_alloc_nozero(sizeof(struct range)); @@ -763,7 +765,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, // "Encode: %llx->%llx into %llx(<-%llx) at idx=%u " // "taint_len=%u shape=%u attr=%u\n", // o_pattern, pattern, repl, changed_val, idx, taint_len, - // h->shape + 1, attr); + // hshape, attr); //#ifdef CMPLOG_SOLVE_TRANSFORM // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3 @@ -845,7 +847,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u64 b_val, o_b_val, mask; u8 bytes; - switch (SHAPE_BYTES(h->shape)) { + switch (hshape) { case 0: case 1: @@ -924,7 +926,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, s64 diff = pattern - b_val; s64 o_diff = o_pattern - o_b_val; /* fprintf(stderr, "DIFF1 idx=%03u shape=%02u %llx-%llx=%lx\n", idx, - h->shape + 1, o_pattern, o_b_val, o_diff); + hshape, o_pattern, o_b_val, o_diff); fprintf(stderr, "DIFF1 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff); */ if (diff == o_diff && diff) { @@ -953,7 +955,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, s64 o_diff = o_pattern ^ o_b_val; /* fprintf(stderr, "DIFF2 idx=%03u shape=%02u %llx-%llx=%lx\n", - idx, h->shape + 1, o_pattern, o_b_val, o_diff); + idx, hshape, o_pattern, o_b_val, o_diff); fprintf(stderr, "DIFF2 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff); */ @@ -1002,7 +1004,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } /* fprintf(stderr, "DIFF3 idx=%03u shape=%02u %llx-%llx=%lx\n", - idx, h->shape + 1, o_pattern, o_b_val, o_diff); + idx, hshape, o_pattern, o_b_val, o_diff); fprintf(stderr, "DIFF3 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff); */ @@ -1051,7 +1053,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } /* fprintf(stderr, "DIFF4 idx=%03u shape=%02u %llx-%llx=%lx\n", - idx, h->shape + 1, o_pattern, o_b_val, o_diff); + idx, hshape, o_pattern, o_b_val, o_diff); fprintf(stderr, "DIFF4 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff); */ @@ -1089,7 +1091,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, if ((lvl & LVL1) || attr >= IS_FP_MOD) { - if (SHAPE_BYTES(h->shape) >= 8 && *status != 1) { + if (hshape >= 8 && *status != 1) { // if (its_len >= 8) // fprintf(stderr, @@ -1132,7 +1134,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - if (SHAPE_BYTES(h->shape) >= 4 && *status != 1) { + if (hshape >= 4 && *status != 1) { // if (its_len >= 4 && (attr <= 1 || attr >= 8)) // fprintf(stderr, @@ -1173,7 +1175,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - if (SHAPE_BYTES(h->shape) >= 2 && *status != 1) { + if (hshape >= 2 && *status != 1) { if (its_len >= 2 && ((*buf_16 == (u16)pattern && *o_buf_16 == (u16)o_pattern) || @@ -1244,7 +1246,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - if (!(attr & (IS_GREATER | IS_LESSER)) || SHAPE_BYTES(h->shape) < 4) { + if (!(attr & (IS_GREATER | IS_LESSER)) || hshape < 4) { return 0; @@ -1272,7 +1274,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, if (attr & IS_GREATER) { - if (SHAPE_BYTES(h->shape) == 4 && its_len >= 4) { + if (hshape == 4 && its_len >= 4) { float *f = (float *)&repl; float g = *f; @@ -1280,7 +1282,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u32 *r = (u32 *)&g; repl_new = (u32)*r; - } else if (SHAPE_BYTES(h->shape) == 8 && its_len >= 8) { + } else if (hshape == 8 && its_len >= 8) { double *f = (double *)&repl; double g = *f; @@ -1307,7 +1309,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } else { - if (SHAPE_BYTES(h->shape) == 4) { + if (hshape == 4) { float *f = (float *)&repl; float g = *f; @@ -1315,7 +1317,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u32 *r = (u32 *)&g; repl_new = (u32)*r; - } else if (SHAPE_BYTES(h->shape) == 8) { + } else if (hshape == 8) { double *f = (double *)&repl; double g = *f; @@ -1342,7 +1344,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } // transform double to float, llvm likes to do that internally ... - if (SHAPE_BYTES(h->shape) == 8 && its_len >= 4) { + if (hshape == 8 && its_len >= 4) { double *f = (double *)&repl; float g = (float)*f; @@ -1353,7 +1355,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, memcpy(((char *)&repl_new) + 4, (char *)&g, 4); #endif changed_val = repl_new; - h->shape = 3; // modify shape + hshape = 4; // modify shape // fprintf(stderr, "DOUBLE2FLOAT %llx\n", repl_new); @@ -1361,12 +1363,12 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, afl, h, pattern, repl_new, o_pattern, changed_val, 16, idx, taint_len, orig_buf, buf, cbuf, len, 1, lvl, status))) { - h->shape = 7; // recover shape + hshape = 8; // recover shape return 1; } - h->shape = 7; // recover shape + hshape = 7; // recover shape } @@ -1428,14 +1430,13 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, u8 *r = (u8 *)&repl; u8 backup[16]; u32 its_len = MIN(len - idx, taint_len); - u32 shape = h->shape + 1; #if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) size_t off = 0; #else - size_t off = 16 - shape; + size_t off = 16 - hshape; #endif - if (its_len >= shape) { + if (its_len >= hshape) { #ifdef _DEBUG fprintf(stderr, "TestUN: %u>=%u (len=%u idx=%u attr=%u off=%lu) (%u) ", @@ -1462,18 +1463,18 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, fprintf(stderr, "\n"); #endif - if (!memcmp(ptr, p + off, shape) && !memcmp(o_ptr, o_p + off, shape)) { + if (!memcmp(ptr, p + off, hshape) && !memcmp(o_ptr, o_p + off, hshape)) { - memcpy(backup, ptr, shape); - memcpy(ptr, r + off, shape); + memcpy(backup, ptr, hshape); + memcpy(ptr, r + off, hshape); if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } #ifdef CMPLOG_COMBINE - if (*status == 1) { memcpy(cbuf + idx, r, shape); } + if (*status == 1) { memcpy(cbuf + idx, r, hshape); } #endif - memcpy(ptr, backup, shape); + memcpy(ptr, backup, hshape); #ifdef _DEBUG fprintf(stderr, "Status=%u\n", *status); @@ -1485,8 +1486,8 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, if (do_reverse && *status != 1) { if (unlikely(cmp_extend_encodingN( - afl, h, SWAPN(pattern, (shape << 3)), SWAPN(repl, (shape << 3)), - SWAPN(o_pattern, (shape << 3)), SWAPN(changed_val, (shape << 3)), + afl, h, SWAPN(pattern, (hshape << 3)), SWAPN(repl, (hshape << 3)), + SWAPN(o_pattern, (hshape << 3)), SWAPN(changed_val, (hshape << 3)), attr, idx, taint_len, orig_buf, buf, cbuf, len, 0, lvl, status))) { @@ -1615,6 +1616,8 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, u8 s_v0_inc = 1, s_v1_inc = 1; u8 s_v0_dec = 1, s_v1_dec = 1; + hshape = SHAPE_BYTES(h->shape); + if (h->hits > CMP_MAP_H) { loggeds = CMP_MAP_H; @@ -1626,7 +1629,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } #ifdef WORD_SIZE_64 - switch (SHAPE_BYTES(h->shape)) { + switch (hshape) { case 1: case 2: @@ -1680,7 +1683,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, #ifdef _DEBUG fprintf(stderr, "Handling: %llx->%llx vs %llx->%llx attr=%u shape=%u\n", orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, - SHAPE_BYTES(h->shape)); + hshape); #endif t = taint; @@ -1830,7 +1833,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, "END: %llx->%llx vs %llx->%llx attr=%u i=%u found=%u " "isN=%u size=%u\n", orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, i, found_one, - is_n, SHAPE_BYTES(h->shape)); + is_n, hshape); #endif // If failed, add to dictionary @@ -1841,16 +1844,16 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, #ifdef WORD_SIZE_64 if (unlikely(is_n)) { - try_to_add_to_dictN(afl, s128_v0, SHAPE_BYTES(h->shape)); - try_to_add_to_dictN(afl, s128_v1, SHAPE_BYTES(h->shape)); + try_to_add_to_dictN(afl, s128_v0, hshape); + try_to_add_to_dictN(afl, s128_v1, hshape); } else #endif { - try_to_add_to_dict(afl, o->v0, SHAPE_BYTES(h->shape)); - try_to_add_to_dict(afl, o->v1, SHAPE_BYTES(h->shape)); + try_to_add_to_dict(afl, o->v0, hshape); + try_to_add_to_dict(afl, o->v1, hshape); } @@ -2322,6 +2325,8 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, u32 i, j, idx, have_taint = 1, taint_len, loggeds; u8 status = 0, found_one = 0; + hshape = SHAPE_BYTES(h->shape); + if (h->hits > CMP_MAP_RTN_H) { loggeds = CMP_MAP_RTN_H; @@ -2355,12 +2360,12 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, /* struct cmp_header *hh = &afl->orig_cmp_map->headers[key]; fprintf(stderr, "RTN N hits=%u id=%u shape=%u attr=%u v0=", h->hits, - h->id, h->shape, h->attribute); + h->id, hshape, h->attribute); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v0[j]); fprintf(stderr, " v1="); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v1[j]); fprintf(stderr, "\nRTN O hits=%u id=%u shape=%u attr=%u o0=", - hh->hits, hh->id, hh->shape, hh->attribute); + hh->hits, hh->id, hshape, hh->attribute); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v0[j]); fprintf(stderr, " o1="); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v1[j]); @@ -2401,23 +2406,23 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, #ifdef _DEBUG int w; fprintf(stderr, "key=%u idx=%u len=%u o0=", key, idx, - SHAPE_BYTES(h->shape)); - for (w = 0; w < SHAPE_BYTES(h->shape); ++w) + hshape); + for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", orig_o->v0[w]); fprintf(stderr, " v0="); - for (w = 0; w < SHAPE_BYTES(h->shape); ++w) + for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", o->v0[w]); fprintf(stderr, " o1="); - for (w = 0; w < SHAPE_BYTES(h->shape); ++w) + for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", orig_o->v1[w]); fprintf(stderr, " v1="); - for (w = 0; w < SHAPE_BYTES(h->shape); ++w) + for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", o->v1[w]); fprintf(stderr, "\n"); #endif if (unlikely(rtn_extend_encoding( - afl, o->v0, o->v1, orig_o->v0, orig_o->v1, SHAPE_BYTES(h->shape), + afl, o->v0, o->v1, orig_o->v0, orig_o->v1, hshape, idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) { return 1; @@ -2434,7 +2439,7 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, status = 0; if (unlikely(rtn_extend_encoding( - afl, o->v1, o->v0, orig_o->v1, orig_o->v0, SHAPE_BYTES(h->shape), + afl, o->v1, o->v0, orig_o->v1, orig_o->v0, hshape, idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) { return 1; @@ -2455,8 +2460,8 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, if (unlikely(!afl->pass_stats[key].total)) { - maybe_add_auto(afl, o->v0, SHAPE_BYTES(h->shape)); - maybe_add_auto(afl, o->v1, SHAPE_BYTES(h->shape)); + maybe_add_auto(afl, o->v0, hshape); + maybe_add_auto(afl, o->v1, hshape); } -- cgit 1.4.1 From bf0fbc24ad32b0ec5421f4d2090c85d87aeceee3 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 17 Oct 2021 15:44:48 +0200 Subject: new cmplog add dict strategy --- src/afl-fuzz-redqueen.c | 170 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 131 insertions(+), 39 deletions(-) (limited to 'src') diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 2d610132..30de12c1 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -45,6 +45,23 @@ enum { }; +// add to dictionary enum +// DEFAULT = 1, notTXT = 2, FOUND = 4, notSAME = 8 +enum { + + DICT_ADD_NEVER = 0, + DICT_ADD_NOTFOUND_SAME_TXT = 1, + DICT_ADD_NOTFOUND_SAME = 3, + DICT_ADD_FOUND_SAME_TXT = 5, + DICT_ADD_FOUND_SAME = 7, + DICT_ADD_NOTFOUND_TXT = 9, + DICT_ADD_NOTFOUND = 11, + DICT_ADD_FOUND_TXT = 13, + DICT_ADD_FOUND = 15, + DICT_ADD_ANY = DICT_ADD_FOUND + +}; + // CMPLOG LVL enum { @@ -54,6 +71,8 @@ enum { }; +#define DICT_ADD_STRATEGY DICT_ADD_FOUND_SAME + struct range { u32 start; @@ -1246,11 +1265,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - if (!(attr & (IS_GREATER | IS_LESSER)) || hshape < 4) { - - return 0; - - } + if (!(attr & (IS_GREATER | IS_LESSER)) || hshape < 4) { return 0; } // transform >= to < and <= to > if ((attr & IS_EQUAL) && (attr & (IS_GREATER | IS_LESSER))) { @@ -1487,9 +1502,9 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, if (unlikely(cmp_extend_encodingN( afl, h, SWAPN(pattern, (hshape << 3)), SWAPN(repl, (hshape << 3)), - SWAPN(o_pattern, (hshape << 3)), SWAPN(changed_val, (hshape << 3)), - attr, idx, taint_len, orig_buf, buf, cbuf, len, 0, lvl, - status))) { + SWAPN(o_pattern, (hshape << 3)), + SWAPN(changed_val, (hshape << 3)), attr, idx, taint_len, orig_buf, + buf, cbuf, len, 0, lvl, status))) { return 1; @@ -1682,8 +1697,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, #ifdef _DEBUG fprintf(stderr, "Handling: %llx->%llx vs %llx->%llx attr=%u shape=%u\n", - orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, - hshape); + orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, hshape); #endif t = taint; @@ -1836,26 +1850,39 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, is_n, hshape); #endif - // If failed, add to dictionary - if (!found_one) { - - if (afl->pass_stats[key].total == 0) { + u8 same0 = 0, same1 = 0, result = 1 + 2 + (found_one << 2); + if (o->v0 != orig_o->v0) { same0 = 8; } + if (o->v1 != orig_o->v1) { same1 = 8; } #ifdef WORD_SIZE_64 - if (unlikely(is_n)) { + if (unlikely(is_n)) { - try_to_add_to_dictN(afl, s128_v0, hshape); - try_to_add_to_dictN(afl, s128_v1, hshape); + if (DICT_ADD_STRATEGY >= same0 + result) { - } else + try_to_add_to_dictN(afl, s128_v0, hshape); + + } + + if (DICT_ADD_STRATEGY >= same1 + result) { + + try_to_add_to_dictN(afl, s128_v1, hshape); + + } + + } else #endif - { + { - try_to_add_to_dict(afl, o->v0, hshape); - try_to_add_to_dict(afl, o->v1, hshape); + if (DICT_ADD_STRATEGY >= same0 + result) { - } + try_to_add_to_dict(afl, o->v0, hshape); + + } + + if (DICT_ADD_STRATEGY >= same1 + result) { + + try_to_add_to_dict(afl, o->v1, hshape); } @@ -1885,8 +1912,9 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } -static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl, - u8 *o_pattern, u8 *changed_val, u8 plen, u32 idx, +static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, + struct cmpfn_operands *o, + struct cmpfn_operands *orig_o, u32 idx, u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf, u32 len, u8 lvl, u8 *status) { @@ -1897,9 +1925,50 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl, // (void)(changed_val); //#endif + u8 *pattern, *repl, *o_pattern, *changed_val; + u8 l0, l1, ol0, ol1; + + if (entry == 0) { + + pattern = o->v0; + repl = o->v1; + o_pattern = orig_o->v0; + changed_val = orig_o->v1; + l0 = o->v0_len; + ol0 = orig_o->v0_len; + l1 = o->v1_len; + ol1 = orig_o->v1_len; + + } else { + + pattern = o->v1; + repl = o->v0; + o_pattern = orig_o->v1; + changed_val = orig_o->v0; + l0 = o->v1_len; + ol0 = orig_o->v1_len; + l1 = o->v0_len; + ol1 = orig_o->v0_len; + + } + + if (l0 >= 0x80) { + + l0 -= 0x80; + l1 -= 0x80; + ol0 -= 0x80; + ol1 -= 0x80; + + } else if (l0 == 0 || l1 == 0 || ol0 == 0 || ol1 == 0) { + + l0 = l1 = ol0 = ol1 = hshape; + + } + + u8 lmax = MAX(l0, ol0); u8 save[40]; u32 saved_idx = idx, pre, from = 0, to = 0, i, j; - u32 its_len = MIN((u32)plen, len - idx); + u32 its_len = MIN(MIN(lmax, hshape), len - idx); its_len = MIN(its_len, taint_len); u32 saved_its_len = its_len; @@ -1915,7 +1984,8 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl, (void)(j); #ifdef _DEBUG - fprintf(stderr, "RTN T idx=%u lvl=%02x ", idx, lvl); + fprintf(stderr, "RTN T idx=%u lvl=%02x is_txt=%u shape=%u/%u ", idx, lvl, + o->v0_len >= 0x80 ? 1 : 0, hshape, l0); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_buf[idx + j]); fprintf(stderr, " -> "); @@ -2405,8 +2475,7 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, #ifdef _DEBUG int w; - fprintf(stderr, "key=%u idx=%u len=%u o0=", key, idx, - hshape); + fprintf(stderr, "key=%u idx=%u len=%u o0=", key, idx, hshape); for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", orig_o->v0[w]); fprintf(stderr, " v0="); @@ -2421,9 +2490,9 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, fprintf(stderr, "\n"); #endif - if (unlikely(rtn_extend_encoding( - afl, o->v0, o->v1, orig_o->v0, orig_o->v1, hshape, - idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) { + if (unlikely(rtn_extend_encoding(afl, 0, o, orig_o, idx, taint_len, + orig_buf, buf, cbuf, len, lvl, + &status))) { return 1; @@ -2438,9 +2507,9 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, status = 0; - if (unlikely(rtn_extend_encoding( - afl, o->v1, o->v0, orig_o->v1, orig_o->v0, hshape, - idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) { + if (unlikely(rtn_extend_encoding(afl, 1, o, orig_o, idx, taint_len, + orig_buf, buf, cbuf, len, lvl, + &status))) { return 1; @@ -2455,13 +2524,36 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } - // If failed, add to dictionary - if (!found_one && (lvl & LVL1)) { + // if (unlikely(!afl->pass_stats[key].total)) { + + if (lvl & LVL1) { + + u8 is_txt = 0, l0 = o->v0_len, ol0 = orig_o->v0_len, l1 = o->v1_len, + ol1 = orig_o->v1_len; + if (l0 >= 0x80) { + + is_txt = 1; + l0 -= 0x80; + l1 -= 0x80; + ol0 -= 0x80; + ol1 -= 0x80; + + } + + u8 same0 = 0, same1 = 0, result = 1 + (found_one << 2); + if (!is_txt) result += 2; + if (l0 != ol0 || memcmp(o->v0, orig_o->v0, l0) != 0) { same0 = 8; } + if (l1 != ol1 || memcmp(o->v1, orig_o->v1, l1) != 0) { same1 = 8; } + + if (DICT_ADD_STRATEGY >= same0 + result) { + + maybe_add_auto(afl, o->v0, l0); + + } - if (unlikely(!afl->pass_stats[key].total)) { + if (DICT_ADD_STRATEGY >= same1 + result) { - maybe_add_auto(afl, o->v0, hshape); - maybe_add_auto(afl, o->v1, hshape); + maybe_add_auto(afl, o->v1, l1); } -- cgit 1.4.1 From 462149de642451024199557c7a7bb7ea76e5b8fa Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 17 Oct 2021 20:31:02 +0200 Subject: fix --- instrumentation/afl-compiler-rt.o.c | 18 +------- src/afl-fuzz-redqueen.c | 87 ++++++++++++++++++++++--------------- 2 files changed, 53 insertions(+), 52 deletions(-) (limited to 'src') diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 21772ca0..38beafb7 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -1944,15 +1944,8 @@ void __cmplog_rtn_hook_n(u8 *ptr1, u8 *ptr2, u32 len) { void __cmplog_rtn_hook_strn(u8 *ptr1, u8 *ptr2, u32 len) { /* - u32 i; if (area_is_valid(ptr1, 32) <= 0 || area_is_valid(ptr2, 32) <= 0) return; - fprintf(stderr, "rtn_strn len=%u arg0=", len); - for (i = 0; i < len; i++) - fprintf(stderr, "%02x", ptr1[i]); - fprintf(stderr, " arg1="); - for (i = 0; i < len; i++) - fprintf(stderr, "%02x", ptr2[i]); - fprintf(stderr, "\n"); + fprintf(stderr, "rtn_strn len=%u arg0=%s arg1=%s\n", len, ptr1, ptr2); */ if (likely(!__afl_cmp_map)) return; @@ -2005,15 +1998,8 @@ void __cmplog_rtn_hook_strn(u8 *ptr1, u8 *ptr2, u32 len) { void __cmplog_rtn_hook_str(u8 *ptr1, u8 *ptr2) { /* - u32 i; if (area_is_valid(ptr1, 32) <= 0 || area_is_valid(ptr2, 32) <= 0) return; - fprintf(stderr, "rtn_str arg0="); - for (i = 0; i < len; i++) - fprintf(stderr, "%02x", ptr1[i]); - fprintf(stderr, " arg1="); - for (i = 0; i < len; i++) - fprintf(stderr, "%02x", ptr2[i]); - fprintf(stderr, "\n"); + fprintf(stderr, "rtn_str arg0=%s arg1=%s\n", ptr1, ptr2); */ if (likely(!__afl_cmp_map)) return; diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 30de12c1..65d21b0a 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -1455,25 +1455,25 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, #ifdef _DEBUG fprintf(stderr, "TestUN: %u>=%u (len=%u idx=%u attr=%u off=%lu) (%u) ", - its_len, shape, len, idx, attr, off, do_reverse); + its_len, hshape, len, idx, attr, off, do_reverse); u32 i; u8 *o_r = (u8 *)&changed_val; - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", ptr[i]); fprintf(stderr, "=="); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", p[off + i]); fprintf(stderr, " "); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", o_ptr[i]); fprintf(stderr, "=="); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", o_p[off + i]); fprintf(stderr, " <= "); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", r[off + i]); fprintf(stderr, "<-"); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", o_r[off + i]); fprintf(stderr, "\n"); #endif @@ -1850,39 +1850,46 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, is_n, hshape); #endif - u8 same0 = 0, same1 = 0, result = 1 + 2 + (found_one << 2); - if (o->v0 != orig_o->v0) { same0 = 8; } - if (o->v1 != orig_o->v1) { same1 = 8; } + // we only learn 16 bit + + if (hshape > 1) { + + u8 same0 = 0, same1 = 0, result = 1 + 2 + (found_one << 2); + if (o->v0 != orig_o->v0) { same0 = 8; } + if (o->v1 != orig_o->v1) { same1 = 8; } #ifdef WORD_SIZE_64 - if (unlikely(is_n)) { + if (unlikely(is_n)) { - if (DICT_ADD_STRATEGY >= same0 + result) { + if (DICT_ADD_STRATEGY >= same0 + result) { - try_to_add_to_dictN(afl, s128_v0, hshape); + try_to_add_to_dictN(afl, s128_v0, hshape); - } + } - if (DICT_ADD_STRATEGY >= same1 + result) { + if (DICT_ADD_STRATEGY >= same1 + result) { - try_to_add_to_dictN(afl, s128_v1, hshape); + try_to_add_to_dictN(afl, s128_v1, hshape); - } + } - } else + } else #endif - { + { - if (DICT_ADD_STRATEGY >= same0 + result) { + if (DICT_ADD_STRATEGY >= same0 + result) { - try_to_add_to_dict(afl, o->v0, hshape); + // fprintf(stderr, "add v0 0x%llx\n", o->v0); + try_to_add_to_dict(afl, o->v0, hshape); - } + } - if (DICT_ADD_STRATEGY >= same1 + result) { + if (DICT_ADD_STRATEGY >= same1 + result) { - try_to_add_to_dict(afl, o->v1, hshape); + // fprintf(stderr, "add v1 0x%llx\n", o->v1); + try_to_add_to_dict(afl, o->v1, hshape); + + } } @@ -2428,18 +2435,22 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } /* - struct cmp_header *hh = &afl->orig_cmp_map->headers[key]; - fprintf(stderr, "RTN N hits=%u id=%u shape=%u attr=%u v0=", h->hits, - h->id, hshape, h->attribute); - for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v0[j]); - fprintf(stderr, " v1="); - for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v1[j]); - fprintf(stderr, "\nRTN O hits=%u id=%u shape=%u attr=%u o0=", - hh->hits, hh->id, hshape, hh->attribute); - for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v0[j]); - fprintf(stderr, " o1="); - for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v1[j]); - fprintf(stderr, "\n"); + struct cmp_header *hh = &afl->orig_cmp_map->headers[key]; + fprintf(stderr, "RTN N hits=%u id=%u shape=%u attr=%u v0=", h->hits, h->id, + hshape, h->attribute); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", o->v0[j]); + fprintf(stderr, " v1="); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", o->v1[j]); + fprintf(stderr, "\nRTN O hits=%u id=%u shape=%u attr=%u o0=", hh->hits, + hh->id, hshape, hh->attribute); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", orig_o->v0[j]); + fprintf(stderr, " o1="); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", orig_o->v1[j]); + fprintf(stderr, "\n"); */ t = taint; @@ -2547,12 +2558,14 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, if (DICT_ADD_STRATEGY >= same0 + result) { + // fprintf(stderr, "add v0 [%u]\"%s\"\n", l0, o->v0); maybe_add_auto(afl, o->v0, l0); } if (DICT_ADD_STRATEGY >= same1 + result) { + // fprintf(stderr, "add v1 [%u]\"%s\"\n", l1, o->v1); maybe_add_auto(afl, o->v1, l1); } @@ -2727,11 +2740,13 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { if (afl->shm.cmp_map->headers[k].type == CMP_TYPE_INS) { + // fprintf(stderr, "INS %u\n", k); afl->stage_max += MIN((u32)(afl->shm.cmp_map->headers[k].hits), (u32)CMP_MAP_H); } else { + // fprintf(stderr, "RTN %u\n", k); afl->stage_max += MIN((u32)(afl->shm.cmp_map->headers[k].hits), (u32)CMP_MAP_RTN_H); -- cgit 1.4.1 From 72d10fee407f32d4041573d1906a047a67277eff Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 18 Oct 2021 10:03:39 +0200 Subject: dict enhancement --- src/afl-fuzz-redqueen.c | 64 ++++++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 25 deletions(-) (limited to 'src') diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 65d21b0a..10bcd63d 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -1853,41 +1853,48 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, // we only learn 16 bit + if (hshape > 1) { - u8 same0 = 0, same1 = 0, result = 1 + 2 + (found_one << 2); + u8 same0 = 0, same1 = 0, same2 = 0, same3 = 0, + result = 1 + 2 + (found_one << 2); if (o->v0 != orig_o->v0) { same0 = 8; } if (o->v1 != orig_o->v1) { same1 = 8; } + if (o->v0 != o->v1) { same2 = 8; } + if (orig_o->v0 != orig_o->v1) { same3 = 8; } + + if (!(same0 && same1) && !same2 && !same3) { #ifdef WORD_SIZE_64 - if (unlikely(is_n)) { + if (unlikely(is_n)) { - if (DICT_ADD_STRATEGY >= same0 + result) { + if (DICT_ADD_STRATEGY >= same0 + result) { - try_to_add_to_dictN(afl, s128_v0, hshape); + try_to_add_to_dictN(afl, s128_v0, hshape); - } + } - if (DICT_ADD_STRATEGY >= same1 + result) { + if (DICT_ADD_STRATEGY >= same1 + result) { - try_to_add_to_dictN(afl, s128_v1, hshape); + try_to_add_to_dictN(afl, s128_v1, hshape); - } + } - } else + } else #endif - { + { - if (DICT_ADD_STRATEGY >= same0 + result) { + if (DICT_ADD_STRATEGY >= same0 + result) { - // fprintf(stderr, "add v0 0x%llx\n", o->v0); - try_to_add_to_dict(afl, o->v0, hshape); + // fprintf(stderr, "add v0 0x%llx\n", o->v0); + try_to_add_to_dict(afl, o->v0, hshape); - } + } - if (DICT_ADD_STRATEGY >= same1 + result) { + if (DICT_ADD_STRATEGY >= same1 + result) { + + // fprintf(stderr, "add v1 0x%llx\n", o->v1); + try_to_add_to_dict(afl, o->v1, hshape); - // fprintf(stderr, "add v1 0x%llx\n", o->v1); - try_to_add_to_dict(afl, o->v1, hshape); + } } @@ -2551,22 +2558,29 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } - u8 same0 = 0, same1 = 0, result = 1 + (found_one << 2); + u8 same0 = 0, same1 = 0, same2 = 0, same3 = 0, + result = 1 + (found_one << 2); if (!is_txt) result += 2; if (l0 != ol0 || memcmp(o->v0, orig_o->v0, l0) != 0) { same0 = 8; } if (l1 != ol1 || memcmp(o->v1, orig_o->v1, l1) != 0) { same1 = 8; } + if (l0 != l1 || memcmp(o->v0, o->v1, l0) != 0) { same2 = 8; } + if (ol0 != ol1 || memcmp(orig_o->v0, orig_o->v1, l0) != 0) { same3 = 8; } - if (DICT_ADD_STRATEGY >= same0 + result) { + if (!(same0 && same1) && !same2 && !same3) { - // fprintf(stderr, "add v0 [%u]\"%s\"\n", l0, o->v0); - maybe_add_auto(afl, o->v0, l0); + if (DICT_ADD_STRATEGY >= same0 + result) { - } + // fprintf(stderr, "add v0 [%u]\"%s\"\n", l0, o->v0); + maybe_add_auto(afl, o->v0, l0); - if (DICT_ADD_STRATEGY >= same1 + result) { + } + + if (DICT_ADD_STRATEGY >= same1 + result) { - // fprintf(stderr, "add v1 [%u]\"%s\"\n", l1, o->v1); - maybe_add_auto(afl, o->v1, l1); + // fprintf(stderr, "add v1 [%u]\"%s\"\n", l1, o->v1); + maybe_add_auto(afl, o->v1, l1); + + } } -- cgit 1.4.1 From 699df8f8ce4cc5de56510f72ebff611d26710557 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 18 Oct 2021 11:11:56 +0200 Subject: fix --- src/afl-fuzz-redqueen.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 10bcd63d..30bad557 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -1860,18 +1860,18 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, if (o->v0 != o->v1) { same2 = 8; } if (orig_o->v0 != orig_o->v1) { same3 = 8; } - if (!(same0 && same1) && !same2 && !same3) { + if (!same2 && !same3) { #ifdef WORD_SIZE_64 if (unlikely(is_n)) { - if (DICT_ADD_STRATEGY >= same0 + result) { + if (!(!same0 && same1) && DICT_ADD_STRATEGY >= same0 + result) { try_to_add_to_dictN(afl, s128_v0, hshape); } - if (DICT_ADD_STRATEGY >= same1 + result) { + if (!(same0 && !same1) && DICT_ADD_STRATEGY >= same1 + result) { try_to_add_to_dictN(afl, s128_v1, hshape); @@ -2566,16 +2566,16 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, if (l0 != l1 || memcmp(o->v0, o->v1, l0) != 0) { same2 = 8; } if (ol0 != ol1 || memcmp(orig_o->v0, orig_o->v1, l0) != 0) { same3 = 8; } - if (!(same0 && same1) && !same2 && !same3) { + if (!same2 && !same3) { - if (DICT_ADD_STRATEGY >= same0 + result) { + if (!(same0 && !same1) && DICT_ADD_STRATEGY >= same0 + result) { // fprintf(stderr, "add v0 [%u]\"%s\"\n", l0, o->v0); maybe_add_auto(afl, o->v0, l0); } - if (DICT_ADD_STRATEGY >= same1 + result) { + if (!(!same0 && same1) && DICT_ADD_STRATEGY >= same1 + result) { // fprintf(stderr, "add v1 [%u]\"%s\"\n", l1, o->v1); maybe_add_auto(afl, o->v1, l1); -- cgit 1.4.1 From 45d668a671316821c3f9793381cb54956b535491 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 18 Oct 2021 13:17:07 +0200 Subject: better ui banner --- docs/Changelog.md | 1 + include/afl-fuzz.h | 1 - src/afl-fuzz-init.c | 37 ------------------------------------- src/afl-fuzz-stats.c | 49 +++++++++++++++++++++++++++++-------------------- src/afl-fuzz.c | 17 +++++++++++++---- 5 files changed, 43 insertions(+), 62 deletions(-) (limited to 'src') diff --git a/docs/Changelog.md b/docs/Changelog.md index 6db013cf..63896622 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -17,6 +17,7 @@ sending a mail to . coverage being detected. thanks to Collin May for reporting! - fix -n dumb mode (nobody should use this) - fix stability issue with LTO and cmplog + - better banner - frida_mode: David Carlier added Android support :) - afl-showmap, afl-tmin and afl-analyze now honor persistent mode for more speed. thanks to dloffre-snl for reporting! diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 4b19e698..eaf55fb8 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1130,7 +1130,6 @@ void get_core_count(afl_state_t *); void fix_up_sync(afl_state_t *); void check_asan_opts(afl_state_t *); void check_binary(afl_state_t *, u8 *); -void fix_up_banner(afl_state_t *, u8 *); void check_if_tty(afl_state_t *); void setup_signal_handlers(void); void save_cmdline(afl_state_t *, u32, char **); diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 9bb25785..9c45f08a 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -2815,43 +2815,6 @@ void check_binary(afl_state_t *afl, u8 *fname) { } -/* Trim and possibly create a banner for the run. */ - -void fix_up_banner(afl_state_t *afl, u8 *name) { - - if (!afl->use_banner) { - - if (afl->sync_id) { - - afl->use_banner = afl->sync_id; - - } else { - - u8 *trim = strrchr(name, '/'); - if (!trim) { - - afl->use_banner = name; - - } else { - - afl->use_banner = trim + 1; - - } - - } - - } - - if (strlen(afl->use_banner) > 32) { - - u8 *tmp = ck_alloc(36); - sprintf(tmp, "%.32s...", afl->use_banner); - afl->use_banner = tmp; - - } - -} - /* Check if we're on TTY. */ void check_if_tty(afl_state_t *afl) { diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 870ba69a..0c06232b 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -441,9 +441,10 @@ void show_stats(afl_state_t *afl) { u64 cur_ms; u32 t_bytes, t_bits; - u32 banner_len, banner_pad; - u8 tmp[256]; - u8 time_tmp[64]; + static u8 banner[128]; + u32 banner_len, banner_pad; + u8 tmp[256]; + u8 time_tmp[64]; u8 val_buf[8][STRINGIFY_VAL_SIZE_MAX]; #define IB(i) (val_buf[(i)]) @@ -656,26 +657,34 @@ void show_stats(afl_state_t *afl) { } /* Let's start by drawing a centered banner. */ + if (unlikely(!banner[0])) { - banner_len = (afl->crash_mode ? 24 : 22) + strlen(VERSION) + - strlen(afl->use_banner) + strlen(afl->power_name) + 3 + 5; - banner_pad = (79 - banner_len) / 2; - memset(tmp, ' ', banner_pad); + char *si = ""; + if (afl->sync_id) { si = afl->sync_id; } + memset(banner, 0, sizeof(banner)); + banner_len = (afl->crash_mode ? 20 : 18) + strlen(VERSION) + strlen(si) + + strlen(afl->power_name) + 4 + 6; -#ifdef HAVE_AFFINITY - sprintf( - tmp + banner_pad, - "%s " cLCY VERSION cLGN " (%s) " cPIN "[%s]" cBLU " {%d}", - afl->crash_mode ? cPIN "peruvian were-rabbit" : cYEL "american fuzzy lop", - afl->use_banner, afl->power_name, afl->cpu_aff); -#else - sprintf( - tmp + banner_pad, "%s " cLCY VERSION cLGN " (%s) " cPIN "[%s]", - afl->crash_mode ? cPIN "peruvian were-rabbit" : cYEL "american fuzzy lop", - afl->use_banner, afl->power_name); -#endif /* HAVE_AFFINITY */ + if (strlen(afl->use_banner) + banner_len > 75) { + + afl->use_banner += (strlen(afl->use_banner) + banner_len) - 76; + memset(afl->use_banner, '.', 3); + + } + + banner_len += strlen(afl->use_banner); + banner_pad = (79 - banner_len) / 2; + memset(banner, ' ', banner_pad); + + sprintf(banner + banner_pad, + "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s]", + afl->crash_mode ? cPIN "peruvian were-rabbit" + : cYEL "american fuzzy lop", + si, afl->use_banner, afl->power_name); + + } - SAYF("\n%s\n", tmp); + SAYF("\n%s\n", banner); /* "Handy" shortcuts for drawing boxes... */ diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 92a37697..26886a4f 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1189,7 +1189,17 @@ int main(int argc, char **argv_orig, char **envp) { } - if (afl->sync_id) { fix_up_sync(afl); } + if (afl->sync_id) { + + if (strlen(afl->sync_id) > 24) { + + FATAL("sync_id max length is 24 characters"); + + } + + fix_up_sync(afl); + + } if (!strcmp(afl->in_dir, afl->out_dir)) { @@ -1218,6 +1228,8 @@ int main(int argc, char **argv_orig, char **envp) { if (unlikely(afl->afl_env.afl_statsd)) { statsd_setup_format(afl); } + if (!afl->use_banner) { afl->use_banner = argv[optind]; } + if (strchr(argv[optind], '/') == NULL && !afl->unicorn_mode) { WARNF(cLRD @@ -1486,9 +1498,6 @@ int main(int argc, char **argv_orig, char **envp) { } save_cmdline(afl, argc, argv); - - fix_up_banner(afl, argv[optind]); - check_if_tty(afl); if (afl->afl_env.afl_force_ui) { afl->not_on_tty = 0; } -- cgit 1.4.1 From 77a63d8ccfd4b409c35227e174f1d6e809256e41 Mon Sep 17 00:00:00 2001 From: Andrea Fioraldi Date: Tue, 19 Oct 2021 13:59:31 +0200 Subject: execs field in filenames --- src/afl-fuzz-bitmap.c | 5 +++-- src/afl-fuzz-init.c | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 0ae4d607..316067e4 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -317,8 +317,9 @@ u8 *describe_op(afl_state_t *afl, u8 new_bits, size_t max_description_len) { } - sprintf(ret + strlen(ret), ",time:%llu", - get_cur_time() + afl->prev_run_time - afl->start_time); + sprintf(ret + strlen(ret), ",time:%llu,execs:%llu", + get_cur_time() + afl->prev_run_time - afl->start_time, + afl->fsrv.total_execs); if (afl->current_custom_fuzz && afl->current_custom_fuzz->afl_custom_describe) { diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 9bb25785..0fa8e6ec 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -1325,8 +1325,8 @@ void pivot_inputs(afl_state_t *afl) { } - nfn = alloc_printf("%s/queue/id:%06u,time:0,orig:%s", afl->out_dir, id, - use_name); + nfn = alloc_printf("%s/queue/id:%06u,time:0,execs:%llu,orig:%s", afl->out_dir, id, + afl->fsrv.total_execs, use_name); #else -- cgit 1.4.1 From 0bc3367b55b2f08c7c2588576af27567044dc0b6 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 19 Oct 2021 14:46:15 +0200 Subject: remove race condition --- src/afl-forkserver.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 54f510c4..94ca3009 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -610,12 +610,24 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (!time_ms) { - if (fsrv->fsrv_pid > 0) { kill(fsrv->fsrv_pid, fsrv->kill_signal); } + s32 tmp_pid = fsrv->fsrv_pid; + if (tmp_pid > 0) { + + kill(tmp_pid, fsrv->kill_signal); + fsrv->fsrv_pid = 1; + + } } else if (time_ms > fsrv->init_tmout) { fsrv->last_run_timed_out = 1; - if (fsrv->fsrv_pid > 0) { kill(fsrv->fsrv_pid, fsrv->kill_signal); } + s32 tmp_pid = fsrv->fsrv_pid; + if (tmp_pid > 0) { + + kill(tmp_pid, fsrv->kill_signal); + fsrv->fsrv_pid = 1; + + } } else { @@ -1248,7 +1260,14 @@ fsrv_run_result_t afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, /* If there was no response from forkserver after timeout seconds, we kill the child. The forkserver should inform us afterwards */ - if (fsrv->child_pid > 0) { kill(fsrv->child_pid, fsrv->kill_signal); } + s32 tmp_pid = srv->child_pid; + if (tmp_pid > 0) { + + kill(tmp_pid, fsrv->kill_signal); + fsrv->child_pid = -1 + + } + fsrv->last_run_timed_out = 1; if (read(fsrv->fsrv_st_fd, &fsrv->child_status, 4) < 4) { exec_ms = 0; } -- cgit 1.4.1 From 90786e2ce9970c52e661c0fe290cb78a1a063004 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 19 Oct 2021 15:20:59 +0200 Subject: fix --- src/afl-forkserver.c | 4 ++-- test/test-pre.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 94ca3009..71667262 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -1260,11 +1260,11 @@ fsrv_run_result_t afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, /* If there was no response from forkserver after timeout seconds, we kill the child. The forkserver should inform us afterwards */ - s32 tmp_pid = srv->child_pid; + s32 tmp_pid = fsrv->child_pid; if (tmp_pid > 0) { kill(tmp_pid, fsrv->kill_signal); - fsrv->child_pid = -1 + fsrv->child_pid = -1; } diff --git a/test/test-pre.sh b/test/test-pre.sh index 7819da47..e12d95be 100755 --- a/test/test-pre.sh +++ b/test/test-pre.sh @@ -88,6 +88,8 @@ unset AFL_QEMU_PERSISTENT_GPR unset AFL_QEMU_PERSISTENT_RET unset AFL_QEMU_PERSISTENT_HOOK unset AFL_QEMU_PERSISTENT_CNT +unset AFL_QEMU_PERSISTENT_MEM +unset AFL_QEMU_PERSISTENT_EXITS unset AFL_CUSTOM_MUTATOR_LIBRARY unset AFL_PYTHON_MODULE unset AFL_PRELOAD -- cgit 1.4.1 From e03897a0703673aa0de7772185a5b5230641cb6a Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 23 Oct 2021 20:54:24 +0200 Subject: fix timeout bug in afl tools --- docs/Changelog.md | 6 ++++-- src/afl-analyze.c | 12 ++++++++++++ src/afl-fuzz-init.c | 4 ++-- src/afl-showmap.c | 13 +++++++++++++ src/afl-tmin.c | 12 ++++++++++++ 5 files changed, 43 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/docs/Changelog.md b/docs/Changelog.md index 63896622..04b2fb2e 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -19,8 +19,10 @@ sending a mail to . - fix stability issue with LTO and cmplog - better banner - frida_mode: David Carlier added Android support :) - - afl-showmap, afl-tmin and afl-analyze now honor persistent mode - for more speed. thanks to dloffre-snl for reporting! + - afl-showmap, afl-tmin and afl-analyze: + - honor persistent mode for more speed. thanks to dloffre-snl for + reporting! + - fix bug where targets are not killed on timeouts - Prevent accidently killing non-afl/fuzz services when aborting afl-showmap and other tools. - afl-cc: diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 8295488d..09b01541 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -120,6 +120,17 @@ static u8 count_class_lookup[256] = { #undef TIMES8 #undef TIMES4 +static void kill_child() { + + if (fsrv.child_pid > 0) { + + kill(fsrv.child_pid, fsrv.kill_signal); + fsrv.child_pid = -1; + + } + +} + static void classify_counts(u8 *mem) { u32 i = map_size; @@ -1053,6 +1064,7 @@ int main(int argc, char **argv_orig, char **envp) { fsrv.target_path = find_binary(argv[optind]); fsrv.trace_bits = afl_shm_init(&shm, map_size, 0); detect_file_args(argv + optind, fsrv.out_file, &use_stdin); + signal(SIGALRM, kill_child); if (qemu_mode) { diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index f0e1a80d..1170715f 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -1325,8 +1325,8 @@ void pivot_inputs(afl_state_t *afl) { } - nfn = alloc_printf("%s/queue/id:%06u,time:0,execs:%llu,orig:%s", afl->out_dir, id, - afl->fsrv.total_execs, use_name); + nfn = alloc_printf("%s/queue/id:%06u,time:0,execs:%llu,orig:%s", + afl->out_dir, id, afl->fsrv.total_execs, use_name); #else diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 5df07bf2..3a244c04 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -146,6 +146,17 @@ static const u8 count_class_binary[256] = { #undef TIMES8 #undef TIMES4 +static void kill_child() { + + if (fsrv->child_pid > 0) { + + kill(fsrv->child_pid, fsrv->kill_signal); + fsrv->child_pid = -1; + + } + +} + static void classify_counts(afl_forkserver_t *fsrv) { u8 * mem = fsrv->trace_bits; @@ -526,6 +537,8 @@ static void showmap_run_target(afl_forkserver_t *fsrv, char **argv) { } + signal(SIGALRM, kill_child); + setitimer(ITIMER_REAL, &it, NULL); if (waitpid(fsrv->child_pid, &status, 0) <= 0) { FATAL("waitpid() failed"); } diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 4f3a6b80..ce2a0b8f 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -120,6 +120,17 @@ static const u8 count_class_lookup[256] = { #undef TIMES8 #undef TIMES4 +static void kill_child() { + + if (fsrv->child_pid > 0) { + + kill(fsrv->child_pid, fsrv->kill_signal); + fsrv->child_pid = -1; + + } + +} + static sharedmem_t *deinit_shmem(afl_forkserver_t *fsrv, sharedmem_t * shm_fuzz) { @@ -1125,6 +1136,7 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->target_path = find_binary(argv[optind]); fsrv->trace_bits = afl_shm_init(&shm, map_size, 0); detect_file_args(argv + optind, out_file, &fsrv->use_stdin); + signal(SIGALRM, kill_child); if (fsrv->qemu_mode) { -- cgit 1.4.1 From a7ee11a1747347847b06a4226f2800dd780f7c16 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 24 Oct 2021 19:35:58 +0200 Subject: fix --- src/afl-forkserver.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 71667262..c570a2bb 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -603,7 +603,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, /* Wait for the fork server to come up, but don't wait too long. */ rlen = 0; - if (fsrv->exec_tmout) { + if (fsrv->init_tmout) { u32 time_ms = read_s32_timed(fsrv->fsrv_st_fd, &status, fsrv->init_tmout, stop_soon_p); @@ -614,7 +614,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (tmp_pid > 0) { kill(tmp_pid, fsrv->kill_signal); - fsrv->fsrv_pid = 1; + fsrv->fsrv_pid = -1; } @@ -625,7 +625,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (tmp_pid > 0) { kill(tmp_pid, fsrv->kill_signal); - fsrv->fsrv_pid = 1; + fsrv->fsrv_pid = -1; } @@ -1301,7 +1301,7 @@ fsrv_run_result_t afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, } - if (!WIFSTOPPED(fsrv->child_status)) { fsrv->child_pid = 0; } + if (!WIFSTOPPED(fsrv->child_status)) { fsrv->child_pid = -1 ; } fsrv->total_execs++; -- cgit 1.4.1 From 0f49463edec0c019bd098659fa74c58a2d28c439 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 24 Oct 2021 19:41:06 +0200 Subject: fix --- src/afl-forkserver.c | 2 +- src/afl-showmap.c | 9 ++++++--- unicorn_mode/unicornafl | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index c570a2bb..80b295e0 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -1301,7 +1301,7 @@ fsrv_run_result_t afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, } - if (!WIFSTOPPED(fsrv->child_status)) { fsrv->child_pid = -1 ; } + if (!WIFSTOPPED(fsrv->child_status)) { fsrv->child_pid = -1; } fsrv->total_execs++; diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 3a244c04..3826e385 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -77,7 +77,7 @@ static u32 tcnt, highest; /* tuple content information */ static u32 in_len; /* Input data length */ -static u32 map_size = MAP_SIZE; +static u32 map_size = MAP_SIZE, timed_out = 0; static bool quiet_mode, /* Hide non-essential messages? */ edges_only, /* Ignore hit counts? */ @@ -148,6 +148,7 @@ static const u8 count_class_binary[256] = { static void kill_child() { + timed_out = 1; if (fsrv->child_pid > 0) { kill(fsrv->child_pid, fsrv->kill_signal); @@ -373,9 +374,10 @@ static void showmap_run_target_forkserver(afl_forkserver_t *fsrv, u8 *mem, if (!quiet_mode) { - if (fsrv->last_run_timed_out) { + if (timed_out || fsrv->last_run_timed_out) { SAYF(cLRD "\n+++ Program timed off +++\n" cRST); + timed_out = 0; } else if (stop_soon) { @@ -581,9 +583,10 @@ static void showmap_run_target(afl_forkserver_t *fsrv, char **argv) { if (!quiet_mode) { - if (fsrv->last_run_timed_out) { + if (timed_out || fsrv->last_run_timed_out) { SAYF(cLRD "\n+++ Program timed off +++\n" cRST); + timed_out = 0; } else if (stop_soon) { diff --git a/unicorn_mode/unicornafl b/unicorn_mode/unicornafl index 94617f5b..d4915053 160000 --- a/unicorn_mode/unicornafl +++ b/unicorn_mode/unicornafl @@ -1 +1 @@ -Subproject commit 94617f5bee0c08a5cbd1a1aa73f59cd973dfb17f +Subproject commit d4915053d477dd827b3fe4b494173d3fbf9f456e -- cgit 1.4.1 From 0348ede4bc47264473efdac86415b9c805bdda40 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 24 Oct 2021 19:53:07 +0200 Subject: fix gcc warning --- src/afl-cc.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-cc.c b/src/afl-cc.c index e49addc4..77407a98 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -1007,7 +1007,11 @@ static void edit_params(u32 argc, char **argv, char **envp) { } // prevent unnecessary build errors - cc_params[cc_par_cnt++] = "-Wno-unused-command-line-argument"; + if (compiler_mode != GCC_PLUGIN && compiler_mode != GCC) { + + cc_params[cc_par_cnt++] = "-Wno-unused-command-line-argument"; + + } if (preprocessor_only || have_c) { -- cgit 1.4.1 From efec2b5a99d8f229eec4123e28c79a15389fe6bb Mon Sep 17 00:00:00 2001 From: lazymio Date: Mon, 25 Oct 2021 16:05:58 +0200 Subject: Update real_map_size when doing a realloc --- src/afl-fuzz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 26886a4f..34f3377b 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1706,7 +1706,7 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->non_instrumented_mode || afl->fsrv.qemu_mode || afl->fsrv.frida_mode || afl->unicorn_mode) { - map_size = afl->fsrv.map_size = MAP_SIZE; + map_size = afl->fsrv.real_map_size = afl->fsrv.map_size = MAP_SIZE; afl->virgin_bits = ck_realloc(afl->virgin_bits, map_size); afl->virgin_tmout = ck_realloc(afl->virgin_tmout, map_size); afl->virgin_crash = ck_realloc(afl->virgin_crash, map_size); -- cgit 1.4.1 From b4e328803c4e61c81fdf0e0412f013fcf0e72e0d Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 26 Oct 2021 18:10:57 +0200 Subject: variants --- include/cmplog.h | 2 +- src/afl-fuzz-redqueen.c | 93 ++++++++++++++++++++++++++++++++++++------------- 2 files changed, 70 insertions(+), 25 deletions(-) (limited to 'src') diff --git a/include/cmplog.h b/include/cmplog.h index 88aa0a61..b95a8e43 100644 --- a/include/cmplog.h +++ b/include/cmplog.h @@ -33,7 +33,7 @@ #define CMPLOG_LVL_MAX 3 #define CMP_MAP_W 65536 -#define CMP_MAP_H 64 +#define CMP_MAP_H 32 #define CMP_MAP_RTN_H (CMP_MAP_H / 4) #define SHAPE_BYTES(x) (x + 1) diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 30bad557..6002703e 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -28,6 +28,8 @@ #include "afl-fuzz.h" #include "cmplog.h" +#define VARIANT 0 + //#define _DEBUG //#define CMPLOG_INTROSPECTION @@ -461,10 +463,10 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, fprintf( f, "Colorization: fname=%s len=%u ms=%llu result=%u execs=%u found=%llu " - "taint=%u\n", + "taint=%u ascii=%u auto_extra_before=%u\n", afl->queue_cur->fname, len, get_cur_time() - start_time, afl->queue_cur->colorized, afl->stage_cur, new_hit_cnt - orig_hit_cnt, - positions); + positions, afl->queue_cur->is_ascii ? 1 : 0, afl->a_extras_cnt); #ifndef _DEBUG if (afl->not_on_tty) { fclose(f); } @@ -1854,24 +1856,34 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, if (hshape > 1) { u8 same0 = 0, same1 = 0, same2 = 0, same3 = 0, - result = 1 + 2 + (found_one << 2); + result = 1 + (found_one << 2); if (o->v0 != orig_o->v0) { same0 = 8; } if (o->v1 != orig_o->v1) { same1 = 8; } if (o->v0 != o->v1) { same2 = 8; } if (orig_o->v0 != orig_o->v1) { same3 = 8; } + if (check_if_text_buf((u8 *)o->v0, hshape) < hshape) same0 += 2; + if (check_if_text_buf((u8 *)o->v1, hshape) < hshape) same1 += 2; if (!same2 && !same3) { #ifdef WORD_SIZE_64 if (unlikely(is_n)) { - if (!(!same0 && same1) && DICT_ADD_STRATEGY >= same0 + result) { + if ( + #if VARIANT == 1 + !(!same0 && same1) && + #endif + DICT_ADD_STRATEGY >= same0 + result) { try_to_add_to_dictN(afl, s128_v0, hshape); } - if (!(same0 && !same1) && DICT_ADD_STRATEGY >= same1 + result) { + if ( + #if VARIANT == 1 + !(same0 && !same1) && + #endif + DICT_ADD_STRATEGY >= same1 + result) { try_to_add_to_dictN(afl, s128_v1, hshape); @@ -1882,14 +1894,22 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, #endif { - if (DICT_ADD_STRATEGY >= same0 + result) { + if ( +#if VARIANT == 1 + !(!same0 && same1) && +#endif + DICT_ADD_STRATEGY >= same0 + result) { // fprintf(stderr, "add v0 0x%llx\n", o->v0); try_to_add_to_dict(afl, o->v0, hshape); } - if (DICT_ADD_STRATEGY >= same1 + result) { + if ( +#if VARIANT == 1 + !(same0 && !same1) && +#endif + DICT_ADD_STRATEGY >= same1 + result) { // fprintf(stderr, "add v1 0x%llx\n", o->v1); try_to_add_to_dict(afl, o->v1, hshape); @@ -1966,14 +1986,17 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - if (l0 >= 0x80) { + if (l0 >= 0x80 || ol0 >= 0x80) { l0 -= 0x80; l1 -= 0x80; ol0 -= 0x80; ol1 -= 0x80; - } else if (l0 == 0 || l1 == 0 || ol0 == 0 || ol1 == 0) { + } + + if (l0 == 0 || l1 == 0 || ol0 == 0 || ol1 == 0 || l0 > 31 || l1 > 31 || + ol0 > 31 || ol1 > 31) { l0 = l1 = ol0 = ol1 = hshape; @@ -2059,10 +2082,10 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - //#ifdef CMPLOG_SOLVE_TRANSFORM - if (*status == 1) return 0; + // transform solving + if (afl->cmplog_enable_transform && (lvl & LVL3)) { u32 toupper = 0, tolower = 0, xor = 0, arith = 0, tohex = 0, fromhex = 0; @@ -2548,7 +2571,7 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, u8 is_txt = 0, l0 = o->v0_len, ol0 = orig_o->v0_len, l1 = o->v1_len, ol1 = orig_o->v1_len; - if (l0 >= 0x80) { + if (l0 >= 0x80 || ol0 >= 0x80) { is_txt = 1; l0 -= 0x80; @@ -2558,29 +2581,51 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } + if (l0 == 0 || l1 == 0 || ol0 == 0 || ol1 == 0 || l0 > 31 || l1 > 31 || + ol0 > 31 || ol1 > 31) { + + l0 = l1 = ol0 = ol1 = hshape; + + } + u8 same0 = 0, same1 = 0, same2 = 0, same3 = 0, result = 1 + (found_one << 2); - if (!is_txt) result += 2; if (l0 != ol0 || memcmp(o->v0, orig_o->v0, l0) != 0) { same0 = 8; } if (l1 != ol1 || memcmp(o->v1, orig_o->v1, l1) != 0) { same1 = 8; } if (l0 != l1 || memcmp(o->v0, o->v1, l0) != 0) { same2 = 8; } if (ol0 != ol1 || memcmp(orig_o->v0, orig_o->v1, l0) != 0) { same3 = 8; } + if (!is_txt && check_if_text_buf((u8 *)&o->v0, l0) < l0) { same0 += 2; } + if (!is_txt && check_if_text_buf((u8 *)&o->v1, l1) < l1) { same1 += 2; } - if (!same2 && !same3) { - - if (!(same0 && !same1) && DICT_ADD_STRATEGY >= same0 + result) { + if ( +#if VARIANT < 3 + !(same0 && !same1) && +#endif +#if VARIANT == 4 + (!same2 && same3) +#else + (DICT_ADD_STRATEGY >= same0 + result) +#endif + ) { - // fprintf(stderr, "add v0 [%u]\"%s\"\n", l0, o->v0); - maybe_add_auto(afl, o->v0, l0); + // fprintf(stderr, "add v0 [%u]\"%s\"\n", l0, o->v0); + maybe_add_auto(afl, o->v0, l0); - } - - if (!(!same0 && same1) && DICT_ADD_STRATEGY >= same1 + result) { + } - // fprintf(stderr, "add v1 [%u]\"%s\"\n", l1, o->v1); - maybe_add_auto(afl, o->v1, l1); + if ( +#if VARIANT < 3 + !(!same0 && same1) && +#endif +#if VARIANT == 4 + (same2 && !same3) +#else + (DICT_ADD_STRATEGY >= same1 + result) +#endif + ) { - } + // fprintf(stderr, "add v1 [%u]\"%s\"\n", l1, o->v1); + maybe_add_auto(afl, o->v1, l1); } -- cgit 1.4.1 From 245e91eb27f1572e00c6664477d588c540c30ed6 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 26 Oct 2021 21:30:08 +0200 Subject: fix --- src/afl-fuzz-redqueen.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 6002703e..800a0753 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -28,7 +28,7 @@ #include "afl-fuzz.h" #include "cmplog.h" -#define VARIANT 0 +#define VARIANT 4 //#define _DEBUG //#define CMPLOG_INTROSPECTION @@ -1861,8 +1861,8 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, if (o->v1 != orig_o->v1) { same1 = 8; } if (o->v0 != o->v1) { same2 = 8; } if (orig_o->v0 != orig_o->v1) { same3 = 8; } - if (check_if_text_buf((u8 *)o->v0, hshape) < hshape) same0 += 2; - if (check_if_text_buf((u8 *)o->v1, hshape) < hshape) same1 += 2; + if (check_if_text_buf((u8 *)&o->v0, hshape) < hshape) same0 += 2; + if (check_if_text_buf((u8 *)&o->v1, hshape) < hshape) same1 += 2; if (!same2 && !same3) { -- cgit 1.4.1 From 9a11c8ac24f8b2c2dcd78aa3a27e5416a717ce4f Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 26 Oct 2021 22:06:01 +0200 Subject: fix --- src/afl-fuzz-redqueen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 800a0753..928cab59 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -28,7 +28,7 @@ #include "afl-fuzz.h" #include "cmplog.h" -#define VARIANT 4 +#define VARIANT 0 //#define _DEBUG //#define CMPLOG_INTROSPECTION -- cgit 1.4.1 From e2ca7a63972ef3c245395bea7bb89d873ad757af Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 28 Oct 2021 18:02:57 +0200 Subject: variant 6 --- instrumentation/afl-compiler-rt.o.c | 44 ++++++------- src/afl-fuzz-redqueen.c | 126 +++++++++++++++++++++++++++++------- 2 files changed, 125 insertions(+), 45 deletions(-) (limited to 'src') diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 5caf57b3..17724e74 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -22,6 +22,10 @@ #include "cmplog.h" #include "llvm-alternative-coverage.h" +#define XXH_INLINE_ALL +#include "xxhash.h" +#undef XXH_INLINE_ALL + #include #include #include @@ -154,6 +158,12 @@ static void at_exit(int signal) { } +#ifdef WORD_SIZE_64 + #define default_hash(a, b) XXH64(a, b, HASH_CONST) +#else + #define default_hash(a, b) XXH64(a, b, HASH_CONST) +#endif + /* Uninspired gcc plugin instrumentation */ void __afl_trace(const u32 x) { @@ -1499,8 +1509,7 @@ void __cmplog_ins_hook1(uint8_t arg1, uint8_t arg2, uint8_t attr) { if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1530,8 +1539,7 @@ void __cmplog_ins_hook2(uint16_t arg1, uint16_t arg2, uint8_t attr) { if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1569,8 +1577,7 @@ void __cmplog_ins_hook4(uint32_t arg1, uint32_t arg2, uint8_t attr) { if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1608,8 +1615,7 @@ void __cmplog_ins_hook8(uint64_t arg1, uint64_t arg2, uint8_t attr) { if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1652,8 +1658,7 @@ void __cmplog_ins_hookN(uint128_t arg1, uint128_t arg2, uint8_t attr, if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1696,8 +1701,7 @@ void __cmplog_ins_hook16(uint128_t arg1, uint128_t arg2, uint8_t attr) { if (likely(!__afl_cmp_map)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1802,8 +1806,8 @@ void __sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) { for (uint64_t i = 0; i < cases[0]; i++) { uintptr_t k = (uintptr_t)__builtin_return_address(0) + i; - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & + (CMP_MAP_W - 1)); u32 hits; @@ -1901,8 +1905,7 @@ void __cmplog_rtn_hook_n(u8 *ptr1, u8 *ptr2, u64 len) { // fprintf(stderr, "RTN2 %u\n", l); uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits, reset = 1; @@ -1955,8 +1958,7 @@ void __cmplog_rtn_hook_strn(u8 *ptr1, u8 *ptr2, u64 len) { // fprintf(stderr, "RTN2 %u\n", l); uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits, reset = 1; @@ -2011,8 +2013,7 @@ void __cmplog_rtn_hook_str(u8 *ptr1, u8 *ptr2) { // fprintf(stderr, "RTN2 %u\n", l); uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits, reset = 1; @@ -2075,8 +2076,7 @@ void __cmplog_rtn_hook(u8 *ptr1, u8 *ptr2) { // fprintf(stderr, "RTN2 %u\n", len); uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits, reset = 1; diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 928cab59..b5dc23c2 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -28,7 +28,7 @@ #include "afl-fuzz.h" #include "cmplog.h" -#define VARIANT 0 +#define VARIANT 6 //#define _DEBUG //#define CMPLOG_INTROSPECTION @@ -1855,6 +1855,43 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, // we only learn 16 bit + if (hshape > 1) { +#if VARIANT == 6 + if (!found_one || afl->queue_cur->is_ascii) { + + #ifdef WORD_SIZE_64 + if (unlikely(is_n)) { + + if (!found_one || + check_if_text_buf((u8 *)&s128_v0, SHAPE_BYTES(h->shape)) == + SHAPE_BYTES(h->shape)) + try_to_add_to_dictN(afl, s128_v0, SHAPE_BYTES(h->shape)); + if (!found_one || + check_if_text_buf((u8 *)&s128_v1, SHAPE_BYTES(h->shape)) == + SHAPE_BYTES(h->shape)) + try_to_add_to_dictN(afl, s128_v1, SHAPE_BYTES(h->shape)); + + } else + + #endif + { + + if (!memcmp((u8 *)&o->v0, (u8 *)&orig_o->v0, SHAPE_BYTES(h->shape)) && + (!found_one || + check_if_text_buf((u8 *)&o->v0, SHAPE_BYTES(h->shape)) == + SHAPE_BYTES(h->shape))) + try_to_add_to_dict(afl, o->v0, SHAPE_BYTES(h->shape)); + if (!memcmp((u8 *)&o->v1, (u8 *)&orig_o->v1, SHAPE_BYTES(h->shape)) && + (!found_one || + check_if_text_buf((u8 *)&o->v1, SHAPE_BYTES(h->shape)) == + SHAPE_BYTES(h->shape))) + try_to_add_to_dict(afl, o->v1, SHAPE_BYTES(h->shape)); + + } + + } + +#else + u8 same0 = 0, same1 = 0, same2 = 0, same3 = 0, result = 1 + (found_one << 2); if (o->v0 != orig_o->v0) { same0 = 8; } @@ -1866,13 +1903,13 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, if (!same2 && !same3) { -#ifdef WORD_SIZE_64 + #ifdef WORD_SIZE_64 if (unlikely(is_n)) { if ( - #if VARIANT == 1 + #if VARIANT == 1 !(!same0 && same1) && - #endif + #endif DICT_ADD_STRATEGY >= same0 + result) { try_to_add_to_dictN(afl, s128_v0, hshape); @@ -1880,9 +1917,9 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } if ( - #if VARIANT == 1 + #if VARIANT == 1 !(same0 && !same1) && - #endif + #endif DICT_ADD_STRATEGY >= same1 + result) { try_to_add_to_dictN(afl, s128_v1, hshape); @@ -1891,13 +1928,13 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } else -#endif + #endif { if ( -#if VARIANT == 1 + #if VARIANT == 1 !(!same0 && same1) && -#endif + #endif DICT_ADD_STRATEGY >= same0 + result) { // fprintf(stderr, "add v0 0x%llx\n", o->v0); @@ -1906,9 +1943,9 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } if ( -#if VARIANT == 1 + #if VARIANT == 1 !(same0 && !same1) && -#endif + #endif DICT_ADD_STRATEGY >= same1 + result) { // fprintf(stderr, "add v1 0x%llx\n", o->v1); @@ -1920,6 +1957,8 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } +#endif + } cmp_fuzz_next_iter: @@ -2567,6 +2606,44 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, // if (unlikely(!afl->pass_stats[key].total)) { +#if VARIANT == 6 + if ((!found_one && (lvl & LVL1)) || afl->queue_cur->is_ascii) { + + // if (unlikely(!afl->pass_stats[key].total)) { + + u32 shape_len = SHAPE_BYTES(h->shape); + u32 v0_len = shape_len, v1_len = shape_len; + if (afl->queue_cur->is_ascii || + check_if_text_buf((u8 *)&o->v0, shape_len) == shape_len) { + + if (strlen(o->v0)) v0_len = strlen(o->v0); + + } + + if (afl->queue_cur->is_ascii || + check_if_text_buf((u8 *)&o->v1, shape_len) == shape_len) { + + if (strlen(o->v1)) v1_len = strlen(o->v1); + + } + + // fprintf(stderr, "SHOULD: found:%u ascii:%u text?%u:%u %u:%s %u:%s \n", + // found_one, afl->queue_cur->is_ascii, check_if_text_buf((u8 *)&o->v0, + // shape_len), check_if_text_buf((u8 *)&o->v1, shape_len), v0_len, + // o->v0, v1_len, o->v1); + + if (!memcmp(o->v0, orig_o->v0, v0_len) || + (!found_one || check_if_text_buf((u8 *)&o->v0, v0_len) == v0_len)) + maybe_add_auto(afl, o->v0, v0_len); + if (!memcmp(o->v1, orig_o->v1, v1_len) || + (!found_one || check_if_text_buf((u8 *)&o->v1, v1_len) == v1_len)) + maybe_add_auto(afl, o->v1, v1_len); + + //} + + } + +#else if (lvl & LVL1) { u8 is_txt = 0, l0 = o->v0_len, ol0 = orig_o->v0_len, l1 = o->v1_len, @@ -2598,14 +2675,14 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, if (!is_txt && check_if_text_buf((u8 *)&o->v1, l1) < l1) { same1 += 2; } if ( -#if VARIANT < 3 + #if VARIANT < 3 !(same0 && !same1) && -#endif -#if VARIANT == 4 + #endif + #if VARIANT == 4 (!same2 && same3) -#else + #else (DICT_ADD_STRATEGY >= same0 + result) -#endif + #endif ) { // fprintf(stderr, "add v0 [%u]\"%s\"\n", l0, o->v0); @@ -2614,14 +2691,14 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } if ( -#if VARIANT < 3 + #if VARIANT < 3 !(!same0 && same1) && -#endif -#if VARIANT == 4 + #endif + #if VARIANT == 4 (same2 && !same3) -#else + #else (DICT_ADD_STRATEGY >= same1 + result) -#endif + #endif ) { // fprintf(stderr, "add v1 [%u]\"%s\"\n", l1, o->v1); @@ -2631,6 +2708,8 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } +#endif + rtn_fuzz_next_iter: afl->stage_cur++; @@ -2966,9 +3045,10 @@ exit_its: if (f) { fprintf(f, - "Cmplog: fname=%s len=%u ms=%llu result=%u finds=%llu entries=%u\n", + "Cmplog: fname=%s len=%u ms=%llu result=%u finds=%llu entries=%u " + "auto_extra_after=%u\n", afl->queue_cur->fname, len, get_cur_time() - start_time, r, - new_hit_cnt - orig_hit_cnt, cmp_locations); + new_hit_cnt - orig_hit_cnt, cmp_locations, afl->a_extras_cnt); #ifndef _DEBUG if (afl->not_on_tty) { fclose(f); } -- cgit 1.4.1 From 36d8f979724de64ad986acd3b9fe8933e99950c4 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 28 Oct 2021 15:38:50 +0200 Subject: gcc instrumentation opt --- src/afl-cc.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/afl-cc.c b/src/afl-cc.c index 77407a98..ed6390ce 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -423,6 +423,8 @@ static void edit_params(u32 argc, char **argv, char **envp) { char *fplugin_arg = alloc_printf("-fplugin=%s/afl-gcc-pass.so", obj_path); cc_params[cc_par_cnt++] = fplugin_arg; + cc_params[cc_par_cnt++] = "-fno-if-conversion"; + cc_params[cc_par_cnt++] = "-fno-if-conversion2"; } -- cgit 1.4.1 From c64735df9e87f2ee15ea32208be85e481c78814b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 31 Oct 2021 11:55:36 +0100 Subject: help gcc --- src/afl-cc.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src') diff --git a/src/afl-cc.c b/src/afl-cc.c index ed6390ce..e7f08aac 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -737,6 +737,14 @@ static void edit_params(u32 argc, char **argv, char **envp) { } + if ((compiler_mode == GCC || compiler_mode == GCC_PLUGIN) && + !strncmp(cur, "-stdlib=", 8)) { + + if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } + continue; + + } + if ((!strncmp(cur, "-fsanitize=fuzzer-", strlen("-fsanitize=fuzzer-")) || !strncmp(cur, "-fsanitize-coverage", strlen("-fsanitize-coverage"))) && (strncmp(cur, "sanitize-coverage-allow", -- cgit 1.4.1 From 639d108512027bda6e7baf4a96f9d4e058e3c1d3 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 31 Oct 2021 12:29:45 +0100 Subject: add cmplog compatability check --- include/types.h | 2 + instrumentation/afl-compiler-rt.o.c | 9 ++- src/afl-forkserver.c | 11 +++ src/afl-fuzz-redqueen.c | 144 +----------------------------------- 4 files changed, 22 insertions(+), 144 deletions(-) (limited to 'src') diff --git a/include/types.h b/include/types.h index 7b94fb83..063e6bbd 100644 --- a/include/types.h +++ b/include/types.h @@ -46,6 +46,7 @@ typedef uint128_t u128; #define FS_ERROR_SHM_OPEN 4 #define FS_ERROR_SHMAT 8 #define FS_ERROR_MMAP 16 +#define FS_ERROR_OLD_CMPLOG 32 /* Reporting options */ #define FS_OPT_ENABLED 0x80000001 @@ -53,6 +54,7 @@ typedef uint128_t u128; #define FS_OPT_SNAPSHOT 0x20000000 #define FS_OPT_AUTODICT 0x10000000 #define FS_OPT_SHDMEM_FUZZ 0x01000000 +#define FS_OPT_NEWCMPLOG 0x02000000 #define FS_OPT_OLD_AFLPP_WORKAROUND 0x0f000000 // FS_OPT_MAX_MAPSIZE is 8388608 = 0x800000 = 2^23 = 1 << 22 #define FS_OPT_MAX_MAPSIZE ((0x00fffffeU >> 1) + 1) diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 17724e74..6ee86add 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -679,7 +679,7 @@ static void __afl_start_snapshots(void) { /* Phone home and tell the parent that we're OK. If parent isn't there, assume we're not running in forkserver mode and just execute program. */ - status |= (FS_OPT_ENABLED | FS_OPT_SNAPSHOT); + status |= (FS_OPT_ENABLED | FS_OPT_SNAPSHOT | FS_OPT_NEWCMPLOG); if (__afl_sharedmem_fuzzing != 0) status |= FS_OPT_SHDMEM_FUZZ; if (__afl_map_size <= FS_OPT_MAX_MAPSIZE) status |= (FS_OPT_SET_MAPSIZE(__afl_map_size) | FS_OPT_MAPSIZE); @@ -945,7 +945,12 @@ static void __afl_start_forkserver(void) { } if (__afl_sharedmem_fuzzing != 0) { status_for_fsrv |= FS_OPT_SHDMEM_FUZZ; } - if (status_for_fsrv) { status_for_fsrv |= (FS_OPT_ENABLED); } + if (status_for_fsrv) { + + status_for_fsrv |= (FS_OPT_ENABLED | FS_OPT_NEWCMPLOG); + + } + memcpy(tmp, &status_for_fsrv, 4); /* Phone home and tell the parent that we're OK. If parent isn't there, diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 54f510c4..da036d2f 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -342,6 +342,11 @@ static void report_error_and_exit(int error) { "the fuzzing target reports that the mmap() call to the shared " "memory failed."); break; + case FS_ERROR_OLD_CMPLOG: + FATAL( + "the -c cmplog target was instrumented with an too old afl++ " + "version, you need to recompile it."); + break; default: FATAL("unknown error code %d from fuzzing target!", error); @@ -651,6 +656,12 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if ((status & FS_OPT_OLD_AFLPP_WORKAROUND) == FS_OPT_OLD_AFLPP_WORKAROUND) status = (status & 0xf0ffffff); + if ((status & FS_OPT_NEWCMPLOG) == 0 && fsrv->cmplog_binary) { + + report_error_and_exit(FS_ERROR_OLD_CMPLOG); + + } + if ((status & FS_OPT_SNAPSHOT) == FS_OPT_SNAPSHOT) { fsrv->snapshot = 1; diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index b5dc23c2..c97080c7 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -28,8 +28,6 @@ #include "afl-fuzz.h" #include "cmplog.h" -#define VARIANT 6 - //#define _DEBUG //#define CMPLOG_INTROSPECTION @@ -1855,10 +1853,9 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, // we only learn 16 bit + if (hshape > 1) { -#if VARIANT == 6 if (!found_one || afl->queue_cur->is_ascii) { - #ifdef WORD_SIZE_64 +#ifdef WORD_SIZE_64 if (unlikely(is_n)) { if (!found_one || @@ -1872,7 +1869,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } else - #endif +#endif { if (!memcmp((u8 *)&o->v0, (u8 *)&orig_o->v0, SHAPE_BYTES(h->shape)) && @@ -1890,75 +1887,6 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } -#else - - u8 same0 = 0, same1 = 0, same2 = 0, same3 = 0, - result = 1 + (found_one << 2); - if (o->v0 != orig_o->v0) { same0 = 8; } - if (o->v1 != orig_o->v1) { same1 = 8; } - if (o->v0 != o->v1) { same2 = 8; } - if (orig_o->v0 != orig_o->v1) { same3 = 8; } - if (check_if_text_buf((u8 *)&o->v0, hshape) < hshape) same0 += 2; - if (check_if_text_buf((u8 *)&o->v1, hshape) < hshape) same1 += 2; - - if (!same2 && !same3) { - - #ifdef WORD_SIZE_64 - if (unlikely(is_n)) { - - if ( - #if VARIANT == 1 - !(!same0 && same1) && - #endif - DICT_ADD_STRATEGY >= same0 + result) { - - try_to_add_to_dictN(afl, s128_v0, hshape); - - } - - if ( - #if VARIANT == 1 - !(same0 && !same1) && - #endif - DICT_ADD_STRATEGY >= same1 + result) { - - try_to_add_to_dictN(afl, s128_v1, hshape); - - } - - } else - - #endif - { - - if ( - #if VARIANT == 1 - !(!same0 && same1) && - #endif - DICT_ADD_STRATEGY >= same0 + result) { - - // fprintf(stderr, "add v0 0x%llx\n", o->v0); - try_to_add_to_dict(afl, o->v0, hshape); - - } - - if ( - #if VARIANT == 1 - !(same0 && !same1) && - #endif - DICT_ADD_STRATEGY >= same1 + result) { - - // fprintf(stderr, "add v1 0x%llx\n", o->v1); - try_to_add_to_dict(afl, o->v1, hshape); - - } - - } - - } - -#endif - } cmp_fuzz_next_iter: @@ -2606,7 +2534,6 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, // if (unlikely(!afl->pass_stats[key].total)) { -#if VARIANT == 6 if ((!found_one && (lvl & LVL1)) || afl->queue_cur->is_ascii) { // if (unlikely(!afl->pass_stats[key].total)) { @@ -2643,73 +2570,6 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } -#else - if (lvl & LVL1) { - - u8 is_txt = 0, l0 = o->v0_len, ol0 = orig_o->v0_len, l1 = o->v1_len, - ol1 = orig_o->v1_len; - if (l0 >= 0x80 || ol0 >= 0x80) { - - is_txt = 1; - l0 -= 0x80; - l1 -= 0x80; - ol0 -= 0x80; - ol1 -= 0x80; - - } - - if (l0 == 0 || l1 == 0 || ol0 == 0 || ol1 == 0 || l0 > 31 || l1 > 31 || - ol0 > 31 || ol1 > 31) { - - l0 = l1 = ol0 = ol1 = hshape; - - } - - u8 same0 = 0, same1 = 0, same2 = 0, same3 = 0, - result = 1 + (found_one << 2); - if (l0 != ol0 || memcmp(o->v0, orig_o->v0, l0) != 0) { same0 = 8; } - if (l1 != ol1 || memcmp(o->v1, orig_o->v1, l1) != 0) { same1 = 8; } - if (l0 != l1 || memcmp(o->v0, o->v1, l0) != 0) { same2 = 8; } - if (ol0 != ol1 || memcmp(orig_o->v0, orig_o->v1, l0) != 0) { same3 = 8; } - if (!is_txt && check_if_text_buf((u8 *)&o->v0, l0) < l0) { same0 += 2; } - if (!is_txt && check_if_text_buf((u8 *)&o->v1, l1) < l1) { same1 += 2; } - - if ( - #if VARIANT < 3 - !(same0 && !same1) && - #endif - #if VARIANT == 4 - (!same2 && same3) - #else - (DICT_ADD_STRATEGY >= same0 + result) - #endif - ) { - - // fprintf(stderr, "add v0 [%u]\"%s\"\n", l0, o->v0); - maybe_add_auto(afl, o->v0, l0); - - } - - if ( - #if VARIANT < 3 - !(!same0 && same1) && - #endif - #if VARIANT == 4 - (same2 && !same3) - #else - (DICT_ADD_STRATEGY >= same1 + result) - #endif - ) { - - // fprintf(stderr, "add v1 [%u]\"%s\"\n", l1, o->v1); - maybe_add_auto(afl, o->v1, l1); - - } - - } - -#endif - rtn_fuzz_next_iter: afl->stage_cur++; -- cgit 1.4.1 From 701997a2a5226c731c15f8d3f512cd8c91df1612 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 3 Nov 2021 13:33:59 +0100 Subject: fix --- src/afl-fuzz-redqueen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index c97080c7..a9c5625f 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -1383,7 +1383,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - hshape = 7; // recover shape + hshape = 8; // recover shape } -- cgit 1.4.1 From 3670412d2e6841f2cee26e3624a02c08a24671c8 Mon Sep 17 00:00:00 2001 From: yuan Date: Wed, 3 Nov 2021 23:44:37 +0800 Subject: Fix request size & remove redundant code (#1139) * fix request size * fix null terminator index * remove redundant code --- src/afl-common.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/afl-common.c b/src/afl-common.c index db19f0a7..26a0d54b 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -217,11 +217,10 @@ char **get_qemu_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { } - char **new_argv = ck_alloc(sizeof(char *) * (argc + 4)); + char **new_argv = ck_alloc(sizeof(char *) * (argc + 3)); if (unlikely(!new_argv)) { FATAL("Illegal amount of arguments specified"); } memcpy(&new_argv[3], &argv[1], (int)(sizeof(char *)) * (argc - 1)); - new_argv[argc + 3] = NULL; new_argv[2] = *target_path_p; new_argv[1] = "--"; @@ -237,11 +236,10 @@ char **get_qemu_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { char **get_wine_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { - char **new_argv = ck_alloc(sizeof(char *) * (argc + 3)); + char **new_argv = ck_alloc(sizeof(char *) * (argc + 2)); if (unlikely(!new_argv)) { FATAL("Illegal amount of arguments specified"); } memcpy(&new_argv[2], &argv[1], (int)(sizeof(char *)) * (argc - 1)); - new_argv[argc + 2] = NULL; new_argv[1] = *target_path_p; -- cgit 1.4.1 From afc84438c60f7a70464ddc353216e0d9fb0c0b49 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 3 Nov 2021 21:23:03 +0100 Subject: move bitmap and saveauto --- src/afl-fuzz.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 6ec033b2..e06060ab 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2226,13 +2226,12 @@ int main(int argc, char **argv_orig, char **envp) { } - write_bitmap(afl); - save_auto(afl); - stop_fuzzing: afl->force_ui_update = 1; // ensure the screen is reprinted show_stats(afl); // print the screen one last time + write_bitmap(afl); + save_auto(afl); SAYF(CURSOR_SHOW cLRD "\n\n+++ Testing aborted %s +++\n" cRST, afl->stop_soon == 2 ? "programmatically" : "by user"); -- cgit 1.4.1 From 6ce3d7fede6b32b522b6cc4403f7c0101cf4a4bc Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 4 Nov 2021 15:53:17 +0100 Subject: add AFL_USE_TSAN --- TODO.md | 1 + docs/Changelog.md | 2 + docs/fuzzing_expert.md | 2 + frida_mode/src/instrument/instrument.c | 4 +- frida_mode/src/instrument/instrument_arm32.c | 2 + frida_mode/src/instrument/instrument_arm64.c | 1 + frida_mode/src/instrument/instrument_x64.c | 54 ++++++++++++++------------ frida_mode/src/instrument/instrument_x86.c | 1 + frida_mode/src/main.c | 7 ++-- frida_mode/src/prefetch.c | 6 +-- frida_mode/src/seccomp/seccomp_callback.c | 32 ++++++++------- frida_mode/src/seccomp/seccomp_filter.c | 6 +-- include/envs.h | 1 + instrumentation/SanitizerCoverageLTO.so.cc | 3 +- instrumentation/SanitizerCoveragePCGUARD.so.cc | 3 +- instrumentation/afl-llvm-pass.so.cc | 3 +- qemu_mode/libcompcov/libcompcov.so.c | 25 +++++++++--- src/afl-as.c | 1 + src/afl-cc.c | 9 +++++ utils/aflpp_driver/aflpp_qemu_driver.c | 2 +- 20 files changed, 106 insertions(+), 59 deletions(-) (limited to 'src') diff --git a/TODO.md b/TODO.md index 1d4270b4..30676312 100644 --- a/TODO.md +++ b/TODO.md @@ -2,6 +2,7 @@ ## TODO + - AFL_USE_TSAN to docs/env_variables.md after work over - screen update during input2stage - better autodetection of shifting runtime timeout values - Update afl->pending_not_fuzzed for MOpt diff --git a/docs/Changelog.md b/docs/Changelog.md index 04b2fb2e..cfeb8cc1 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -26,7 +26,9 @@ sending a mail to . - Prevent accidently killing non-afl/fuzz services when aborting afl-showmap and other tools. - afl-cc: + - support llvm IR select instrumentation for default PCGUARD and LTO - fix for shared linking on MacOS + - added AFL_USE_TSAN thread sanitizer support - llvm and LTO mode modified to work with new llvm 14-dev (again) - added the very good grammar mutator "GramaTron" to the custom_mutators diff --git a/docs/fuzzing_expert.md b/docs/fuzzing_expert.md index 96193f88..44ebade4 100644 --- a/docs/fuzzing_expert.md +++ b/docs/fuzzing_expert.md @@ -149,6 +149,8 @@ The following sanitizers have built-in support in AFL++: vulnerabilities - which is however one of the most important and dangerous C++ memory corruption classes! Enabled with `export AFL_USE_CFISAN=1` before compiling. + * TSAN = Thread SANitizer, finds thread race conditions. + Enabled with `export AFL_USE_TSAN=1` before compiling. * LSAN = Leak SANitizer, finds memory leaks in a program. This is not really a security issue, but for developers this can be very valuable. Note that unlike the other sanitizers above this needs diff --git a/frida_mode/src/instrument/instrument.c b/frida_mode/src/instrument/instrument.c index 81d85aa1..eeebe545 100644 --- a/frida_mode/src/instrument/instrument.c +++ b/frida_mode/src/instrument/instrument.c @@ -347,8 +347,8 @@ void instrument_init(void) { #else tid = syscall(SYS_gettid); #endif - instrument_hash_seed = g_get_monotonic_time() ^ - (((guint64)getpid()) << 32) ^ tid; + instrument_hash_seed = + g_get_monotonic_time() ^ (((guint64)getpid()) << 32) ^ tid; } diff --git a/frida_mode/src/instrument/instrument_arm32.c b/frida_mode/src/instrument/instrument_arm32.c index 4b0a648e..395d56c1 100644 --- a/frida_mode/src/instrument/instrument_arm32.c +++ b/frida_mode/src/instrument/instrument_arm32.c @@ -23,7 +23,9 @@ void instrument_coverage_optimize(const cs_insn * instr, } void instrument_coverage_optimize_init(void) { + WARNF("Optimized coverage not supported on this architecture"); + } void instrument_flush(GumStalkerOutput *output) { diff --git a/frida_mode/src/instrument/instrument_arm64.c b/frida_mode/src/instrument/instrument_arm64.c index 80d1d845..358e8e6b 100644 --- a/frida_mode/src/instrument/instrument_arm64.c +++ b/frida_mode/src/instrument/instrument_arm64.c @@ -96,6 +96,7 @@ void instrument_coverage_optimize(const cs_insn * instr, } void instrument_coverage_optimize_init(void) { + } void instrument_flush(GumStalkerOutput *output) { diff --git a/frida_mode/src/instrument/instrument_x64.c b/frida_mode/src/instrument/instrument_x64.c index a7eb650a..19ec81b2 100644 --- a/frida_mode/src/instrument/instrument_x64.c +++ b/frida_mode/src/instrument/instrument_x64.c @@ -4,12 +4,12 @@ #include #if defined(__linux__) -#if !defined(__ANDROID__) -#include -#include -#else -#include -#endif + #if !defined(__ANDROID__) + #include + #include + #else + #include + #endif #endif #include "frida-gumjs.h" @@ -22,13 +22,13 @@ #if defined(__x86_64__) -#ifndef MAP_FIXED_NOREPLACE - #ifdef MAP_EXCL - #define MAP_FIXED_NOREPLACE MAP_EXCL | MAP_FIXED - #else - #define MAP_FIXED_NOREPLACE MAP_FIXED + #ifndef MAP_FIXED_NOREPLACE + #ifdef MAP_EXCL + #define MAP_FIXED_NOREPLACE MAP_EXCL | MAP_FIXED + #else + #define MAP_FIXED_NOREPLACE MAP_FIXED + #endif #endif -#endif gboolean instrument_is_coverage_optimize_supported(void) { @@ -53,15 +53,12 @@ typedef struct { // 0x7ffff6cfb08b: pushf // 0x7ffff6cfb08c: push rsi // 0x7ffff6cfb08d: mov rsi,0x228 - // 0x7ffff6cfb094: xchg QWORD PTR [rip+0x3136a5],rsi # 0x7ffff700e740 - // 0x7ffff6cfb09b: xor rsi,0x451 - // 0x7ffff6cfb0a2: add BYTE PTR [rsi+0x10000],0x1 - // 0x7ffff6cfb0a9: adc BYTE PTR [rsi+0x10000],0x0 - // 0x7ffff6cfb0b0: pop rsi - // 0x7ffff6cfb0b1: popf + // 0x7ffff6cfb094: xchg QWORD PTR [rip+0x3136a5],rsi # + // 0x7ffff700e740 0x7ffff6cfb09b: xor rsi,0x451 0x7ffff6cfb0a2: add + // BYTE PTR [rsi+0x10000],0x1 0x7ffff6cfb0a9: adc BYTE PTR + // [rsi+0x10000],0x0 0x7ffff6cfb0b0: pop rsi 0x7ffff6cfb0b1: popf // 0x7ffff6cfb0b2: lea rsp,[rsp+0x80] - uint8_t lea_rsp_rsp_sub_rz[5]; uint8_t push_fq; uint8_t push_rsi; @@ -160,16 +157,25 @@ static void instrument_coverage_optimize_map_mmap(char * shm_file_path, __afl_area_ptr = NULL; -#if !defined(__ANDROID__) + #if !defined(__ANDROID__) shm_fd = shm_open(shm_file_path, O_RDWR, DEFAULT_PERMISSION); if (shm_fd == -1) { FATAL("shm_open() failed\n"); } -#else + #else shm_fd = open("/dev/ashmem", O_RDWR); if (shm_fd == -1) { FATAL("open() failed\n"); } - if (ioctl(shm_fd, ASHMEM_SET_NAME, shm_file_path) == -1) { FATAL("ioctl(ASHMEM_SET_NAME) failed"); } - if (ioctl(shm_fd, ASHMEM_SET_SIZE, __afl_map_size) == -1) { FATAL("ioctl(ASHMEM_SET_SIZE) failed"); } + if (ioctl(shm_fd, ASHMEM_SET_NAME, shm_file_path) == -1) { -#endif + FATAL("ioctl(ASHMEM_SET_NAME) failed"); + + } + + if (ioctl(shm_fd, ASHMEM_SET_SIZE, __afl_map_size) == -1) { + + FATAL("ioctl(ASHMEM_SET_SIZE) failed"); + + } + + #endif __afl_area_ptr = mmap(address, __afl_map_size, PROT_READ | PROT_WRITE, MAP_FIXED_NOREPLACE | MAP_SHARED, shm_fd, 0); diff --git a/frida_mode/src/instrument/instrument_x86.c b/frida_mode/src/instrument/instrument_x86.c index 1ff5c920..f90c01c2 100644 --- a/frida_mode/src/instrument/instrument_x86.c +++ b/frida_mode/src/instrument/instrument_x86.c @@ -84,6 +84,7 @@ void instrument_coverage_optimize(const cs_insn * instr, } void instrument_coverage_optimize_init(void) { + } void instrument_flush(GumStalkerOutput *output) { diff --git a/frida_mode/src/main.c b/frida_mode/src/main.c index c8183d8f..3599143b 100644 --- a/frida_mode/src/main.c +++ b/frida_mode/src/main.c @@ -126,15 +126,16 @@ static void afl_print_cmdline(void) { g_free(fname); g_free(buffer); #elif defined(__APPLE__) - int idx; + int idx; char **argv = *_NSGetArgv(); - int nargv = *_NSGetArgc(); + int nargv = *_NSGetArgc(); - for (idx = 0; idx < nargv; idx ++) { + for (idx = 0; idx < nargv; idx++) { OKF("AFL - COMMANDLINE: argv[%d] = %s", idx, argv[idx]); } + #endif } diff --git a/frida_mode/src/prefetch.c b/frida_mode/src/prefetch.c index c30ca65c..1ddbd5ed 100644 --- a/frida_mode/src/prefetch.c +++ b/frida_mode/src/prefetch.c @@ -44,8 +44,8 @@ static void gum_afl_stalker_backpatcher_notify(GumStalkerObserver *self, sizeof(prefetch_data->backpatch_data) - prefetch_data->backpatch_size; if (sizeof(gsize) + size > remaining) { return; } - gsize *dst_backpatch_size = (gsize *) - &prefetch_data->backpatch_data[prefetch_data->backpatch_size]; + gsize *dst_backpatch_size = + (gsize *)&prefetch_data->backpatch_data[prefetch_data->backpatch_size]; *dst_backpatch_size = size; prefetch_data->backpatch_size += sizeof(gsize); @@ -117,7 +117,7 @@ static void prefetch_read_patches(void) { remaining = prefetch_data->backpatch_size - offset) { gsize *src_backpatch_data = (gsize *)&prefetch_data->backpatch_data[offset]; - gsize size = *src_backpatch_data; + gsize size = *src_backpatch_data; offset += sizeof(gsize); if (prefetch_data->backpatch_size - offset < size) { diff --git a/frida_mode/src/seccomp/seccomp_callback.c b/frida_mode/src/seccomp/seccomp_callback.c index 4232d842..ac0fb8bb 100644 --- a/frida_mode/src/seccomp/seccomp_callback.c +++ b/frida_mode/src/seccomp/seccomp_callback.c @@ -1,8 +1,8 @@ #if defined(__linux__) && !defined(__ANDROID__) -#if !defined(__MUSL__) - #include -#endif + #if !defined(__MUSL__) + #include + #endif #include #include "seccomp.h" @@ -16,12 +16,13 @@ static void seccomp_callback_filter(struct seccomp_notif * req, GumDebugSymbolDetails details = {0}; if (req->data.nr == SYS_OPENAT) { -#if UINTPTR_MAX == 0xffffffffffffffffu + #if UINTPTR_MAX == 0xffffffffffffffffu seccomp_print("SYS_OPENAT: (%s)\n", (char *)req->data.args[1]); -#endif -#if UINTPTR_MAX == 0xffffffff + #endif + #if UINTPTR_MAX == 0xffffffff seccomp_print("SYS_OPENAT: (%s)\n", (char *)(__u32)req->data.args[1]); -#endif + #endif + } seccomp_print( @@ -31,7 +32,7 @@ static void seccomp_callback_filter(struct seccomp_notif * req, req->data.args[0], req->data.args[1], req->data.args[2], req->data.args[3], req->data.args[4], req->data.args[5]); -#if !defined(__MUSL__) + #if !defined(__MUSL__) seccomp_print("FRAMES: (%u)\n", frames->len); char **syms = backtrace_symbols(frames->items, frames->len); if (syms == NULL) { FATAL("Failed to get symbols"); } @@ -52,23 +53,24 @@ static void seccomp_callback_filter(struct seccomp_notif * req, } free(syms); -#else + #else void **syms = (void **)__builtin_frame_address(0); - void *framep = __builtin_frame_address(1); - int i = 0; + void * framep = __builtin_frame_address(1); + int i = 0; syms = framep; while (syms) { - - framep = *syms; + + framep = *syms; syms = framep; if (!syms) break; - seccomp_print("\%3d. %s\n", i ++, (char *)framep); + seccomp_print("\%3d. %s\n", i++, (char *)framep); } -#endif + + #endif resp->error = 0; resp->val = 0; diff --git a/frida_mode/src/seccomp/seccomp_filter.c b/frida_mode/src/seccomp/seccomp_filter.c index 7ee5ead1..0dcc4cbb 100644 --- a/frida_mode/src/seccomp/seccomp_filter.c +++ b/frida_mode/src/seccomp/seccomp_filter.c @@ -2,9 +2,9 @@ #include #include -#if !defined(__MUSL__) - #include -#endif + #if !defined(__MUSL__) + #include + #endif #include #include #include diff --git a/include/envs.h b/include/envs.h index 61267a0d..25d05539 100644 --- a/include/envs.h +++ b/include/envs.h @@ -203,6 +203,7 @@ static char *afl_environment_variables[] = { "AFL_USE_MSAN", "AFL_USE_TRACE_PC", "AFL_USE_UBSAN", + "AFL_USE_TSAN", "AFL_USE_CFISAN", "AFL_USE_LSAN", "AFL_WINE_PATH", diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index ee8c317e..4e25221a 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -1142,10 +1142,11 @@ bool ModuleSanitizerCoverage::instrumentModule( else { char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", + snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", getenv("AFL_USE_MSAN") ? ", MSAN" : "", + getenv("AFL_USE_TSAN") ? ", TSAN" : "", getenv("AFL_USE_CFISAN") ? ", CFISAN" : "", getenv("AFL_USE_UBSAN") ? ", UBSAN" : ""); OKF("Instrumented %u locations (%u selects) without collisions (%llu " diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index be3f4f49..76bb2448 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -547,10 +547,11 @@ bool ModuleSanitizerCoverage::instrumentModule( else { char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", + snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", getenv("AFL_USE_MSAN") ? ", MSAN" : "", + getenv("AFL_USE_TSAN") ? ", TSAN" : "", getenv("AFL_USE_CFISAN") ? ", CFISAN" : "", getenv("AFL_USE_UBSAN") ? ", UBSAN" : ""); OKF("Instrumented %u locations with no collisions (%s mode) of which are " diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index ecf28f31..9b7e625e 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -956,11 +956,12 @@ bool AFLCoverage::runOnModule(Module &M) { else { char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", + snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", getenv("AFL_USE_MSAN") ? ", MSAN" : "", getenv("AFL_USE_CFISAN") ? ", CFISAN" : "", + getenv("AFL_USE_TSAN") ? ", TSAN" : "", getenv("AFL_USE_UBSAN") ? ", UBSAN" : ""); OKF("Instrumented %d locations (%s mode, ratio %u%%).", inst_blocks, modeline, inst_ratio); diff --git a/qemu_mode/libcompcov/libcompcov.so.c b/qemu_mode/libcompcov/libcompcov.so.c index 24867cda..eba3d80a 100644 --- a/qemu_mode/libcompcov/libcompcov.so.c +++ b/qemu_mode/libcompcov/libcompcov.so.c @@ -42,10 +42,10 @@ #endif /* !__linux__ */ #ifndef likely -# define likely(x) __builtin_expect((!!(x)),1) + #define likely(x) __builtin_expect((!!(x)), 1) #endif #ifndef unlikely -# define unlikely(x) __builtin_expect((!!(x)),0) + #define unlikely(x) __builtin_expect((!!(x)), 0) #endif /* Change this value to tune the compare coverage */ @@ -235,7 +235,12 @@ int strcmp(const char *str1, const char *str2) { int strncmp(const char *str1, const char *str2, size_t len) { - if (unlikely(!__libc_strncmp)) { __libc_strncmp = dlsym(RTLD_NEXT, "strncmp"); } + if (unlikely(!__libc_strncmp)) { + + __libc_strncmp = dlsym(RTLD_NEXT, "strncmp"); + + } + void *retaddr = __builtin_return_address(0); if (__compcov_is_in_bound(retaddr) && @@ -265,7 +270,12 @@ int strncmp(const char *str1, const char *str2, size_t len) { int strcasecmp(const char *str1, const char *str2) { - if (unlikely(!__libc_strcasecmp)) { __libc_strncasecmp = dlsym(RTLD_NEXT, "strcasecmp"); } + if (unlikely(!__libc_strcasecmp)) { + + __libc_strncasecmp = dlsym(RTLD_NEXT, "strcasecmp"); + + } + void *retaddr = __builtin_return_address(0); if (__compcov_is_in_bound(retaddr) && @@ -296,7 +306,12 @@ int strcasecmp(const char *str1, const char *str2) { int strncasecmp(const char *str1, const char *str2, size_t len) { - if (unlikely(!__libc_strncasecmp)) { __libc_strncasecmp = dlsym(RTLD_NEXT, "strncasecmp"); } + if (unlikely(!__libc_strncasecmp)) { + + __libc_strncasecmp = dlsym(RTLD_NEXT, "strncasecmp"); + + } + void *retaddr = __builtin_return_address(0); if (__compcov_is_in_bound(retaddr) && diff --git a/src/afl-as.c b/src/afl-as.c index 7119d630..774340ac 100644 --- a/src/afl-as.c +++ b/src/afl-as.c @@ -521,6 +521,7 @@ static void add_instrumentation(void) { getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", getenv("AFL_USE_MSAN") ? ", MSAN" : "", + getenv("AFL_USE_TSAN") ? ", TSAN" : "", getenv("AFL_USE_UBSAN") ? ", UBSAN" : "", getenv("AFL_USE_LSAN") ? ", LSAN" : ""); diff --git a/src/afl-cc.c b/src/afl-cc.c index e7f08aac..3837459b 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -857,6 +857,14 @@ static void edit_params(u32 argc, char **argv, char **envp) { cc_params[cc_par_cnt++] = "-fsanitize=undefined"; cc_params[cc_par_cnt++] = "-fsanitize-undefined-trap-on-error"; cc_params[cc_par_cnt++] = "-fno-sanitize-recover=all"; + cc_params[cc_par_cnt++] = "-fno-omit-frame-pointer"; + + } + + if (getenv("AFL_USE_TSAN")) { + + cc_params[cc_par_cnt++] = "-fsanitize=thread"; + cc_params[cc_par_cnt++] = "-fno-omit-frame-pointer"; } @@ -1814,6 +1822,7 @@ int main(int argc, char **argv, char **envp) { " AFL_USE_CFISAN: activate control flow sanitizer\n" " AFL_USE_MSAN: activate memory sanitizer\n" " AFL_USE_UBSAN: activate undefined behaviour sanitizer\n" + " AFL_USE_TSAN: activate thread sanitizer\n" " AFL_USE_LSAN: activate leak-checker sanitizer\n"); if (have_gcc_plugin) diff --git a/utils/aflpp_driver/aflpp_qemu_driver.c b/utils/aflpp_driver/aflpp_qemu_driver.c index 99a4c9a8..e47df1e6 100644 --- a/utils/aflpp_driver/aflpp_qemu_driver.c +++ b/utils/aflpp_driver/aflpp_qemu_driver.c @@ -22,7 +22,7 @@ int main(int argc, char **argv) { if (LLVMFuzzerInitialize) LLVMFuzzerInitialize(&argc, &argv); // Do any other expensive one-time initialization here. - if (getenv("AFL_QEMU_DRIVER_NO_HOOK")) { + if (getenv("AFL_QEMU_DRIVER_NO_HOOK") || getenv("AFL_FRIDA_DRIVER_NO_HOOK")) { afl_qemu_driver_stdin_input(); -- cgit 1.4.1 From b34bc6c426a53d8ba9c4f472218be80f1cbf79b5 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Thu, 4 Nov 2021 16:21:26 +0000 Subject: afl-as: missing one format spec (for tsan perharps) --- src/afl-as.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-as.c b/src/afl-as.c index 774340ac..9af272f2 100644 --- a/src/afl-as.c +++ b/src/afl-as.c @@ -517,7 +517,7 @@ static void add_instrumentation(void) { } else { char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", + snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", getenv("AFL_USE_MSAN") ? ", MSAN" : "", -- cgit 1.4.1 From 95067729842c4e3d798dc5fda01138ffb552f494 Mon Sep 17 00:00:00 2001 From: yuawn Date: Fri, 5 Nov 2021 03:09:39 +0000 Subject: unify output format --- src/afl-cc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-cc.c b/src/afl-cc.c index 3837459b..5f77b097 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -2057,7 +2057,7 @@ int main(int argc, char **argv, char **envp) { if ((isatty(2) && !be_quiet) || debug) { SAYF(cCYA - "afl-cc " VERSION cRST + "afl-cc" VERSION cRST " by Michal Zalewski, Laszlo Szekeres, Marc Heuse - mode: %s-%s\n", compiler_mode_string[compiler_mode], ptr); -- cgit 1.4.1 From 9325a4fcbb8eb4ed1d71f93de5301bf1a9a68253 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 6 Nov 2021 10:28:22 +0100 Subject: http->https --- GNUmakefile | 2 +- GNUmakefile.gcc_plugin | 2 +- GNUmakefile.llvm | 2 +- TODO.md | 1 - afl-cmin.bash | 2 +- afl-plot | 2 +- afl-whatsup | 2 +- docs/Changelog.md | 2 +- docs/INSTALL.md | 2 +- docs/best_practices.md | 2 +- docs/interpreting_output.md | 2 +- docs/known_limitations.md | 2 +- docs/sister_projects.md | 12 ++++++------ docs/technical_details.md | 12 ++++++------ frida_mode/Scripting.md | 2 +- include/afl-as.h | 4 ++-- include/afl-fuzz.h | 2 +- include/afl-prealloc.h | 2 +- include/alloc-inl.h | 2 +- include/cmplog.h | 2 +- include/common.h | 2 +- include/config.h | 2 +- include/debug.h | 2 +- include/forkserver.h | 2 +- include/hash.h | 2 +- include/list.h | 2 +- include/sharedmem.h | 2 +- include/snapshot-inl.h | 2 +- include/types.h | 2 +- instrumentation/README.llvm.md | 2 +- instrumentation/afl-compiler-rt.o.c | 2 +- instrumentation/afl-gcc-pass.so.cc | 2 +- instrumentation/afl-llvm-dict2file.so.cc | 2 +- instrumentation/afl-llvm-lto-instrumentation.so.cc | 2 +- instrumentation/afl-llvm-lto-instrumentlist.so.cc | 2 +- instrumentation/afl-llvm-pass.so.cc | 2 +- instrumentation/afl-llvm-rt-lto.o.c | 2 +- instrumentation/cmplog-instructions-pass.cc | 2 +- instrumentation/cmplog-routines-pass.cc | 2 +- instrumentation/cmplog-switches-pass.cc | 2 +- instrumentation/compare-transform-pass.so.cc | 2 +- instrumentation/split-compares-pass.so.cc | 2 +- instrumentation/split-switches-pass.so.cc | 2 +- qemu_mode/build_qemu_support.sh | 2 +- src/afl-analyze.c | 2 +- src/afl-as.c | 4 ++-- src/afl-cc.c | 2 +- src/afl-common.c | 2 +- src/afl-forkserver.c | 8 ++++---- src/afl-fuzz-bitmap.c | 2 +- src/afl-fuzz-cmplog.c | 2 +- src/afl-fuzz-extras.c | 2 +- src/afl-fuzz-init.c | 4 ++-- src/afl-fuzz-mutators.c | 2 +- src/afl-fuzz-one.c | 2 +- src/afl-fuzz-python.c | 2 +- src/afl-fuzz-queue.c | 2 +- src/afl-fuzz-redqueen.c | 2 +- src/afl-fuzz-run.c | 2 +- src/afl-fuzz-state.c | 2 +- src/afl-fuzz-stats.c | 2 +- src/afl-fuzz.c | 2 +- src/afl-gotcpu.c | 2 +- src/afl-ld-lto.c | 2 +- src/afl-performance.c | 2 +- src/afl-sharedmem.c | 2 +- src/afl-showmap.c | 2 +- src/afl-tmin.c | 2 +- test-instr.c | 2 +- unicorn_mode/build_unicorn_support.sh | 2 +- 70 files changed, 85 insertions(+), 86 deletions(-) (limited to 'src') diff --git a/GNUmakefile b/GNUmakefile index 0a6f3950..ad2642f3 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -10,7 +10,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # For Heiko: diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index bce97b2f..ed2725d7 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -17,7 +17,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # #TEST_MMAP=1 PREFIX ?= /usr/local diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index b802ef16..64e5beb2 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -12,7 +12,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # For Heiko: diff --git a/TODO.md b/TODO.md index 30676312..1d4270b4 100644 --- a/TODO.md +++ b/TODO.md @@ -2,7 +2,6 @@ ## TODO - - AFL_USE_TSAN to docs/env_variables.md after work over - screen update during input2stage - better autodetection of shifting runtime timeout values - Update afl->pending_not_fuzzed for MOpt diff --git a/afl-cmin.bash b/afl-cmin.bash index c77dfbc1..e25ddc74 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -11,7 +11,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # This tool tries to find the smallest subset of files in the input directory # that still trigger the full range of instrumentation data points seen in diff --git a/afl-plot b/afl-plot index 87b9caae..1ea1fc55 100755 --- a/afl-plot +++ b/afl-plot @@ -12,7 +12,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # get_abs_path() { diff --git a/afl-whatsup b/afl-whatsup index 9c2564c6..10a52f83 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -12,7 +12,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # This tool summarizes the status of any locally-running synchronized # instances of afl-fuzz. diff --git a/docs/Changelog.md b/docs/Changelog.md index cfeb8cc1..7c77a6bf 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -2760,7 +2760,7 @@ sending a mail to . - Updated the documentation and added notes_for_asan.txt. Based on feedback from Hanno Boeck, Ben Laurie, and others. - - Moved the project to http://lcamtuf.coredump.cx/afl/. + - Moved the project to https://lcamtuf.coredump.cx/afl/. ### Version 0.46b: diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 960de1af..cfa20dea 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -150,4 +150,4 @@ sysctl kern.sysv.shmseg=48 sysctl kern.sysv.shmall=98304 ``` -See [http://www.spy-hill.com/help/apple/SharedMemory.html](http://www.spy-hill.com/help/apple/SharedMemory.html) for documentation for these settings and how to make them permanent. \ No newline at end of file +See [https://www.spy-hill.com/help/apple/SharedMemory.html](https://www.spy-hill.com/help/apple/SharedMemory.html) for documentation for these settings and how to make them permanent. \ No newline at end of file diff --git a/docs/best_practices.md b/docs/best_practices.md index 0708d49d..5d07dd14 100644 --- a/docs/best_practices.md +++ b/docs/best_practices.md @@ -108,7 +108,7 @@ Four steps are required to do this and it also requires quite some knowledge of Follow this document on how to do this: [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md). If `PCGUARD` is used, then you need to follow this guide (needs llvm 12+!): - [http://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation](http://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation) + [https://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation](https://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation) Only exclude those functions from instrumentation that provide no value for coverage - that is if it does not process any fuzz data directly or indirectly (e.g. hash maps, thread management etc.). If however a function directly or indirectly handles fuzz data, then you should not put the function in a deny instrumentation list and rather live with the instability it comes with. diff --git a/docs/interpreting_output.md b/docs/interpreting_output.md index 327a0ac0..4bd705f2 100644 --- a/docs/interpreting_output.md +++ b/docs/interpreting_output.md @@ -56,7 +56,7 @@ Any existing output directory can be also used to resume aborted jobs; try: If you have gnuplot installed, you can also generate some pretty graphs for any active fuzzing task using afl-plot. For an example of how this looks like, -see [http://lcamtuf.coredump.cx/afl/plot/](http://lcamtuf.coredump.cx/afl/plot/). +see [https://lcamtuf.coredump.cx/afl/plot/](https://lcamtuf.coredump.cx/afl/plot/). You can also manually build and install afl-plot-ui, which is a helper utility for showing the graphs generated by afl-plot in a graphical window using GTK. diff --git a/docs/known_limitations.md b/docs/known_limitations.md index 2d8f84a5..a68c0a85 100644 --- a/docs/known_limitations.md +++ b/docs/known_limitations.md @@ -31,6 +31,6 @@ Here are some of the most important caveats for AFL: [https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop](https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop) - Occasionally, sentient machines rise against their creators. If this - happens to you, please consult [http://lcamtuf.coredump.cx/prep/](http://lcamtuf.coredump.cx/prep/). + happens to you, please consult [https://lcamtuf.coredump.cx/prep/](https://lcamtuf.coredump.cx/prep/). Beyond this, see [INSTALL.md](INSTALL.md) for platform-specific tips. diff --git a/docs/sister_projects.md b/docs/sister_projects.md index 5cb3a102..613bc778 100644 --- a/docs/sister_projects.md +++ b/docs/sister_projects.md @@ -15,7 +15,7 @@ instruction manual. Allows fuzz-testing of Python programs. Uses custom instrumentation and its own forkserver. -http://jwilk.net/software/python-afl +https://jwilk.net/software/python-afl ### Go-fuzz (Dmitry Vyukov) @@ -34,7 +34,7 @@ https://github.com/kmcallister/afl.rs Adds AFL-compatible instrumentation to OCaml programs. https://github.com/ocamllabs/opam-repo-dev/pull/23 -http://canopy.mirage.io/Posts/Fuzzing +https://canopy.mirage.io/Posts/Fuzzing ### AFL for GCJ Java and other GCC frontends (-) @@ -54,7 +54,7 @@ some programs to be fuzzed without the fork / execve overhead. (Similar functionality is now available as the "persistent" feature described in [the llvm_mode readme](../instrumentation/README.llvm.md)) -http://llvm.org/docs/LibFuzzer.html +https://llvm.org/docs/LibFuzzer.html ## TriforceAFL (Tim Newsham and Jesse Hertz) @@ -189,7 +189,7 @@ https://github.com/bshastry/afl-sancov Makes it easy to estimate memory usage limits when fuzzing with ASAN or MSAN. -http://jwilk.net/software/recidivm +https://jwilk.net/software/recidivm ### aflize (Jacek Wielemborek) @@ -274,7 +274,7 @@ https://goo.gl/j9EgFf A simple SQL shell designed specifically for fuzzing the underlying library. -http://www.sqlite.org/src/artifact/9e7e273da2030371 +https://www.sqlite.org/src/artifact/9e7e273da2030371 ### Support for Python mutation modules (Christian Holler) @@ -292,7 +292,7 @@ A similar guided approach as applied to fuzzing syscalls: https://github.com/google/syzkaller/wiki/Found-Bugs https://github.com/dvyukov/linux/commit/33787098ffaaa83b8a7ccf519913ac5fd6125931 -http://events.linuxfoundation.org/sites/events/files/slides/AFL%20filesystem%20fuzzing%2C%20Vault%202016_0.pdf +https://events.linuxfoundation.org/sites/events/files/slides/AFL%20filesystem%20fuzzing%2C%20Vault%202016_0.pdf ### Kernel Snapshot Fuzzing using Unicornafl (Security in Telecommunications) diff --git a/docs/technical_details.md b/docs/technical_details.md index b0ca493e..b9d271d9 100644 --- a/docs/technical_details.md +++ b/docs/technical_details.md @@ -161,8 +161,8 @@ features of the underlying data format, as shown in this image: Several practical examples of the results of this algorithm are discussed here: - http://lcamtuf.blogspot.com/2014/11/pulling-jpegs-out-of-thin-air.html - http://lcamtuf.blogspot.com/2014/11/afl-fuzz-nobody-expects-cdata-sections.html + https://lcamtuf.blogspot.com/2014/11/pulling-jpegs-out-of-thin-air.html + https://lcamtuf.blogspot.com/2014/11/afl-fuzz-nobody-expects-cdata-sections.html The synthetic corpus produced by this process is essentially a compact collection of "hmm, this does something new!" input files, and can be used to @@ -323,7 +323,7 @@ value of various fuzzing strategies and optimize their parameters so that they work equally well across a wide range of file types. The strategies used by afl-fuzz are generally format-agnostic and are discussed in more detail here: - http://lcamtuf.blogspot.com/2014/08/binary-fuzzing-strategies-what-works.html + https://lcamtuf.blogspot.com/2014/08/binary-fuzzing-strategies-what-works.html It is somewhat notable that especially early on, most of the work done by `afl-fuzz` is actually highly deterministic, and progresses to random stacked @@ -376,7 +376,7 @@ valid grammar for the tested parser. A discussion of how these features are implemented within afl-fuzz can be found here: - http://lcamtuf.blogspot.com/2015/01/afl-fuzz-making-up-grammar-with.html + https://lcamtuf.blogspot.com/2015/01/afl-fuzz-making-up-grammar-with.html In essence, when basic, typically easily-obtained syntax tokens are combined together in a purely random manner, the instrumentation and the evolutionary @@ -429,7 +429,7 @@ thrown away. A detailed discussion of the value of this approach can be found here: - http://lcamtuf.blogspot.com/2014/11/afl-fuzz-crash-exploration-mode.html + https://lcamtuf.blogspot.com/2014/11/afl-fuzz-crash-exploration-mode.html The method uses instrumentation feedback to explore the state of the crashing program to get past the ambiguous faulting condition and then isolate the @@ -447,7 +447,7 @@ goes through `execve()`, linking, and libc initialization only once, and is then cloned from a stopped process image by leveraging copy-on-write. The implementation is described in more detail here: - http://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html + https://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html The fork server is an integral aspect of the injected instrumentation and simply stops at the first instrumented function to await commands from diff --git a/frida_mode/Scripting.md b/frida_mode/Scripting.md index f6017fad..691b03d1 100644 --- a/frida_mode/Scripting.md +++ b/frida_mode/Scripting.md @@ -302,7 +302,7 @@ Consider the [following](test/js/test2.c) test code... Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ #include diff --git a/include/afl-as.h b/include/afl-as.h index 3c12c68f..2a2e8ad7 100644 --- a/include/afl-as.h +++ b/include/afl-as.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This file houses the assembly-level instrumentation injected into fuzzed programs. The instrumentation stores XORed pairs of data: identifiers of the @@ -396,7 +396,7 @@ static const u8 *main_payload_32 = "\n"; /* The OpenBSD hack is due to lahf and sahf not being recognized by some - versions of binutils: http://marc.info/?l=openbsd-cvs&m=141636589924400 + versions of binutils: https://marc.info/?l=openbsd-cvs&m=141636589924400 The Apple code is a bit different when calling libc functions because they are doing relocations differently from everybody else. We also need diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index eaf55fb8..e73ea1a4 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to diff --git a/include/afl-prealloc.h b/include/afl-prealloc.h index fa6c9b70..87bbb1cc 100644 --- a/include/afl-prealloc.h +++ b/include/afl-prealloc.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/include/alloc-inl.h b/include/alloc-inl.h index c914da5f..0c540330 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This allocator is not designed to resist malicious attackers (the canaries are small and predictable), but provides a robust and portable way to detect diff --git a/include/cmplog.h b/include/cmplog.h index 878ed60c..1c15d2b8 100644 --- a/include/cmplog.h +++ b/include/cmplog.h @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code to handle the shared memory. This is used by the fuzzer as well the other components like afl-tmin, afl-showmap, etc... diff --git a/include/common.h b/include/common.h index 2ca44301..e3997aa4 100644 --- a/include/common.h +++ b/include/common.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Gather some functions common to multiple executables diff --git a/include/config.h b/include/config.h index 4630da0c..3aee9b00 100644 --- a/include/config.h +++ b/include/config.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/include/debug.h b/include/debug.h index f8df5711..feb7f52d 100644 --- a/include/debug.h +++ b/include/debug.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/include/forkserver.h b/include/forkserver.h index c6f7de00..7af01cb2 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code that implements a forkserver. This is used by the fuzzer as well the other components like afl-tmin. diff --git a/include/hash.h b/include/hash.h index 9319ab95..9bb34ff8 100644 --- a/include/hash.h +++ b/include/hash.h @@ -21,7 +21,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/include/list.h b/include/list.h index 7ec81cbe..d49e56da 100644 --- a/include/list.h +++ b/include/list.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This allocator is not designed to resist malicious attackers (the canaries are small and predictable), but provides a robust and portable way to detect diff --git a/include/sharedmem.h b/include/sharedmem.h index fdc947f9..93080d0f 100644 --- a/include/sharedmem.h +++ b/include/sharedmem.h @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code to handle the shared memory. This is used by the fuzzer as well the other components like afl-tmin, afl-showmap, etc... diff --git a/include/snapshot-inl.h b/include/snapshot-inl.h index a18187ef..7234bbaa 100644 --- a/include/snapshot-inl.h +++ b/include/snapshot-inl.h @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/include/types.h b/include/types.h index 7b94fb83..e945f0f5 100644 --- a/include/types.h +++ b/include/types.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/instrumentation/README.llvm.md b/instrumentation/README.llvm.md index 5b1e60cc..dbb604f2 100644 --- a/instrumentation/README.llvm.md +++ b/instrumentation/README.llvm.md @@ -75,7 +75,7 @@ load modules (you'll see "Service unavailable" when loading afl-llvm-pass.so). To solve all your problems, you can grab pre-built binaries for your OS from: - http://llvm.org/releases/download.html + https://llvm.org/releases/download.html ...and then put the bin/ directory from the tarball at the beginning of your $PATH when compiling the feature and building packages later on. You don't need diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 9acab4e7..b2802a29 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -9,7 +9,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/instrumentation/afl-gcc-pass.so.cc b/instrumentation/afl-gcc-pass.so.cc index 3b7eb878..df2b6f2a 100644 --- a/instrumentation/afl-gcc-pass.so.cc +++ b/instrumentation/afl-gcc-pass.so.cc @@ -30,7 +30,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program. If not, see . + along with this program. If not, see . */ diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index 0a3e74b9..7c04c0c5 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -10,7 +10,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This library is plugged into LLVM when invoking clang through afl-clang-lto. diff --git a/instrumentation/afl-llvm-lto-instrumentation.so.cc b/instrumentation/afl-llvm-lto-instrumentation.so.cc index c2f61d34..cd43b437 100644 --- a/instrumentation/afl-llvm-lto-instrumentation.so.cc +++ b/instrumentation/afl-llvm-lto-instrumentation.so.cc @@ -10,7 +10,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This library is plugged into LLVM when invoking clang through afl-clang-lto. diff --git a/instrumentation/afl-llvm-lto-instrumentlist.so.cc b/instrumentation/afl-llvm-lto-instrumentlist.so.cc index ee2e5dd3..cf26f912 100644 --- a/instrumentation/afl-llvm-lto-instrumentlist.so.cc +++ b/instrumentation/afl-llvm-lto-instrumentlist.so.cc @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This library is plugged into LLVM when invoking clang through afl-clang-fast. It tells the compiler to add code roughly equivalent to the bits discussed diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index 9b7e625e..21ce0cf9 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This library is plugged into LLVM when invoking clang through afl-clang-fast. It tells the compiler to add code roughly equivalent to the bits discussed diff --git a/instrumentation/afl-llvm-rt-lto.o.c b/instrumentation/afl-llvm-rt-lto.o.c index e53785ff..eb346157 100644 --- a/instrumentation/afl-llvm-rt-lto.o.c +++ b/instrumentation/afl-llvm-rt-lto.o.c @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index 01a8a637..80af05f0 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -11,7 +11,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index 1e2610f2..01b7a373 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -11,7 +11,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/instrumentation/cmplog-switches-pass.cc b/instrumentation/cmplog-switches-pass.cc index c42d44fe..aa719013 100644 --- a/instrumentation/cmplog-switches-pass.cc +++ b/instrumentation/cmplog-switches-pass.cc @@ -11,7 +11,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index 288e8282..a1239040 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index e63be98c..7c652ca2 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -6,7 +6,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/instrumentation/split-switches-pass.so.cc b/instrumentation/split-switches-pass.so.cc index 82f198aa..1e32a31d 100644 --- a/instrumentation/split-switches-pass.so.cc +++ b/instrumentation/split-switches-pass.so.cc @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index 84f144be..71453a71 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -19,7 +19,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # This script downloads, patches, and builds a version of QEMU with # minor tweaks to allow non-instrumented binaries to be run under diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 09b01541..60cb1434 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 A nifty utility that grabs an input file and takes a stab at explaining its structure by observing how changes to it affect the execution path. diff --git a/src/afl-as.c b/src/afl-as.c index 9af272f2..b644b82a 100644 --- a/src/afl-as.c +++ b/src/afl-as.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 The sole purpose of this wrapper is to preprocess assembly files generated by GCC / clang and inject the instrumentation bits included from afl-as.h. It @@ -101,7 +101,7 @@ static void edit_params(int argc, char **argv) { /* On MacOS X, the Xcode cctool 'as' driver is a bit stale and does not work with the code generated by newer versions of clang that are hand-built - by the user. See the thread here: http://goo.gl/HBWDtn. + by the user. See the thread here: https://goo.gl/HBWDtn. To work around this, when using clang and running without AFL_AS specified, we will actually call 'clang -c' instead of 'as -q' to diff --git a/src/afl-cc.c b/src/afl-cc.c index 5f77b097..8ff241ba 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -11,7 +11,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/src/afl-common.c b/src/afl-common.c index 26a0d54b..ec3b2f3f 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Gather some functions common to multiple executables diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 80b295e0..b1769bfb 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -19,7 +19,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code that implements a forkserver. This is used by the fuzzer as well the other components like afl-tmin. @@ -351,7 +351,7 @@ static void report_error_and_exit(int error) { /* Spins up fork server. The idea is explained here: - http://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html + https://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html In essence, the instrumentation allows us to skip execve(), and just keep cloning a stopped child. So, we just execute once, and then send commands @@ -917,7 +917,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" - " Tip: you can use http://jwilk.net/software/recidivm to " + " Tip: you can use https://jwilk.net/software/recidivm to " "quickly\n" " estimate the required amount of virtual memory for the " "binary.\n\n" @@ -1017,7 +1017,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" - " Tip: you can use http://jwilk.net/software/recidivm to quickly\n" + " Tip: you can use https://jwilk.net/software/recidivm to quickly\n" " estimate the required amount of virtual memory for the " "binary.\n\n" diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 316067e4..f7b59f25 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to diff --git a/src/afl-fuzz-cmplog.c b/src/afl-fuzz-cmplog.c index c2e9c80f..6fc926f0 100644 --- a/src/afl-fuzz-cmplog.c +++ b/src/afl-fuzz-cmplog.c @@ -17,7 +17,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code to handle the shared memory. This is used by the fuzzer as well the other components like afl-tmin, afl-showmap, etc... diff --git a/src/afl-fuzz-extras.c b/src/afl-fuzz-extras.c index 584241d4..0f0fe331 100644 --- a/src/afl-fuzz-extras.c +++ b/src/afl-fuzz-extras.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 1170715f..9262d718 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -974,7 +974,7 @@ void perform_dry_run(afl_state_t *afl) { MSG_ULIMIT_USAGE " /path/to/binary [...] . + See . This is xoshiro256++ 1.0, one of our all-purpose, rock-solid generators. It has excellent (sub-ns) speed, a state (256 bits) that is large diff --git a/src/afl-sharedmem.c b/src/afl-sharedmem.c index 22fe5a62..7fb8f821 100644 --- a/src/afl-sharedmem.c +++ b/src/afl-sharedmem.c @@ -17,7 +17,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code to handle the shared memory. This is used by the fuzzer as well the other components like afl-tmin, afl-showmap, etc... diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 3826e385..23ec0df0 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 A very simple tool that runs the targeted binary and displays the contents of the trace bitmap in a human-readable form. Useful in diff --git a/src/afl-tmin.c b/src/afl-tmin.c index ce2a0b8f..8ce4bdd5 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 A simple test case minimizer that takes an input file and tries to remove as much data as possible while keeping the binary in a crashing state diff --git a/test-instr.c b/test-instr.c index 13d4eb93..eaae50ef 100644 --- a/test-instr.c +++ b/test-instr.c @@ -7,7 +7,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ #include diff --git a/unicorn_mode/build_unicorn_support.sh b/unicorn_mode/build_unicorn_support.sh index 6c376f8d..f9c0be7f 100755 --- a/unicorn_mode/build_unicorn_support.sh +++ b/unicorn_mode/build_unicorn_support.sh @@ -20,7 +20,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # This script downloads, patches, and builds a version of Unicorn with # minor tweaks to allow Unicorn-emulated binaries to be run under -- cgit 1.4.1 From 61039b410c98cb971daff5778b1591d0e5cbfd3e Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 6 Nov 2021 14:29:16 +0100 Subject: better error report for old qemu/frida cmplog --- include/types.h | 1 + src/afl-forkserver.c | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/include/types.h b/include/types.h index 063e6bbd..bca4ff92 100644 --- a/include/types.h +++ b/include/types.h @@ -47,6 +47,7 @@ typedef uint128_t u128; #define FS_ERROR_SHMAT 8 #define FS_ERROR_MMAP 16 #define FS_ERROR_OLD_CMPLOG 32 +#define FS_ERROR_OLD_CMPLOG_QEMU 64 /* Reporting options */ #define FS_OPT_ENABLED 0x80000001 diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 93fd19e1..94c90701 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -347,6 +347,11 @@ static void report_error_and_exit(int error) { "the -c cmplog target was instrumented with an too old afl++ " "version, you need to recompile it."); break; + case FS_ERROR_OLD_CMPLOG_QEMU: + FATAL( + "The AFL++ QEMU/FRIDA loaders are from an older version, for -c you " + "need to recompile it.\n"); + break; default: FATAL("unknown error code %d from fuzzing target!", error); @@ -670,7 +675,15 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if ((status & FS_OPT_NEWCMPLOG) == 0 && fsrv->cmplog_binary) { - report_error_and_exit(FS_ERROR_OLD_CMPLOG); + if (fsrv->qemu_mode || fsrv->frida_mode) { + + report_error_and_exit(FS_ERROR_OLD_CMPLOG_QEMU); + + } else { + + report_error_and_exit(FS_ERROR_OLD_CMPLOG); + + } } -- cgit 1.4.1 From 64704403ca957886b3ad0b55efebaf29906712b8 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 6 Nov 2021 14:54:53 +0100 Subject: better screen update --- src/afl-fuzz-redqueen.c | 70 +++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index a9c5625f..4fa62059 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -84,6 +84,8 @@ struct range { }; static u32 hshape; +static u64 screen_update; +static u64 last_update; static struct range *add_range(struct range *ranges, u32 start, u32 end) { @@ -273,7 +275,6 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, u64 start_time = get_cur_time(); #endif - u32 screen_update; u64 orig_hit_cnt, new_hit_cnt, exec_cksum; orig_hit_cnt = afl->queued_paths + afl->unique_crashes; @@ -282,24 +283,6 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, afl->stage_max = (len << 1); afl->stage_cur = 0; - if (likely(afl->queue_cur->exec_us)) { - - if (likely((100000 / 2) >= afl->queue_cur->exec_us)) { - - screen_update = 100000 / afl->queue_cur->exec_us; - - } else { - - screen_update = 1; - - } - - } else { - - screen_update = 100000; - - } - // in colorization we do not classify counts, hence we have to calculate // the original checksum. if (unlikely(get_exec_checksum(afl, buf, len, &exec_cksum))) { @@ -369,7 +352,7 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, } - if (++afl->stage_cur % screen_update) { show_stats(afl); }; + if (++afl->stage_cur % screen_update == 0) { show_stats(afl); }; } @@ -780,6 +763,13 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u32 its_len = MIN(len - idx, taint_len); + if (afl->fsrv.total_execs - last_update > screen_update) { + + show_stats(afl); + last_update = afl->fsrv.total_execs; + + } + // fprintf(stderr, // "Encode: %llx->%llx into %llx(<-%llx) at idx=%u " // "taint_len=%u shape=%u attr=%u\n", @@ -1438,6 +1428,13 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf, u32 len, u8 do_reverse, u8 lvl, u8 *status) { + if (afl->fsrv.total_execs - last_update > screen_update) { + + show_stats(afl); + last_update = afl->fsrv.total_execs; + + } + u8 *ptr = (u8 *)&buf[idx]; u8 *o_ptr = (u8 *)&orig_buf[idx]; u8 *p = (u8 *)&pattern; @@ -1926,6 +1923,13 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, // (void)(changed_val); //#endif + if (afl->fsrv.total_execs - last_update > screen_update) { + + show_stats(afl); + last_update = afl->fsrv.total_execs; + + } + u8 *pattern, *repl, *o_pattern, *changed_val; u8 l0, l1, ol0, ol1; @@ -2600,6 +2604,23 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { } struct tainted *taint = NULL; + if (likely(afl->queue_cur->exec_us)) { + + if (likely((100000 / 2) >= afl->queue_cur->exec_us)) { + + screen_update = 100000 / afl->queue_cur->exec_us; + + } else { + + screen_update = 1; + + } + + } else { + + screen_update = 100000; + + } if (!afl->queue_cur->taint || !afl->queue_cur->cmplog_colorinput) { @@ -2700,8 +2721,6 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { u64 orig_hit_cnt, new_hit_cnt; u64 orig_execs = afl->fsrv.total_execs; orig_hit_cnt = afl->queued_paths + afl->unique_crashes; - u64 screen_update = 100000 / afl->queue_cur->exec_us, - execs = afl->fsrv.total_execs; afl->stage_name = "input-to-state"; afl->stage_short = "its"; @@ -2783,13 +2802,6 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { } - if (afl->fsrv.total_execs - execs > screen_update) { - - execs = afl->fsrv.total_execs; - show_stats(afl); - - } - } r = 0; -- cgit 1.4.1 From 2ddbaa439ca78b0ae8cc6691d9657f5783b2d5e8 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 6 Nov 2021 16:22:38 +0100 Subject: 30% faster hashing speed --- include/xxhash.h | 3112 +++++++++++++++++++++-------------- instrumentation/afl-compiler-rt.o.c | 2 +- src/afl-performance.c | 6 +- 3 files changed, 1894 insertions(+), 1226 deletions(-) (limited to 'src') diff --git a/include/xxhash.h b/include/xxhash.h index 006d3f3d..0ca2b852 100644 --- a/include/xxhash.h +++ b/include/xxhash.h @@ -32,7 +32,12 @@ * - xxHash homepage: https://www.xxhash.com * - xxHash source repository: https://github.com/Cyan4973/xxHash */ - +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ /* TODO: update */ /* Notice extracted from xxHash homepage: @@ -45,7 +50,7 @@ Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo Name Speed Q.Score Author xxHash 5.4 GB/s 10 CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby +MurmurHash 3a 2.7 GB/s 10 Austin Appleby SpookyHash 2.0 GB/s 10 Bob Jenkins SBox 1.4 GB/s 9 Bret Mulvey Lookup3 1.2 GB/s 9 Bob Jenkins @@ -119,29 +124,78 @@ extern "C" { /* * This part deals with the special case where a unit wants to inline xxHash, - * but "xxhash.h" has previously been included without XXH_INLINE_ALL, such - * as part of some previously included *.h header file. + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. * Without further action, the new include would just be ignored, * and functions would effectively _not_ be inlined (silent failure). * The following macros solve this situation by prefixing all inlined names, * avoiding naming collision with previous inclusions. */ - #ifdef XXH_NAMESPACE - #error "XXH_INLINE_ALL with XXH_NAMESPACE is not supported" - /* - * Note: Alternative: #undef all symbols (it's a pretty large list). - * Without #error: it compiles, but functions are actually not inlined. - */ - #endif +/* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ + #undef XXH_versionNumber +/* XXH32 */ + #undef XXH32 + #undef XXH32_createState + #undef XXH32_freeState + #undef XXH32_reset + #undef XXH32_update + #undef XXH32_digest + #undef XXH32_copyState + #undef XXH32_canonicalFromHash + #undef XXH32_hashFromCanonical +/* XXH64 */ + #undef XXH64 + #undef XXH64_createState + #undef XXH64_freeState + #undef XXH64_reset + #undef XXH64_update + #undef XXH64_digest + #undef XXH64_copyState + #undef XXH64_canonicalFromHash + #undef XXH64_hashFromCanonical +/* XXH3_64bits */ + #undef XXH3_64bits + #undef XXH3_64bits_withSecret + #undef XXH3_64bits_withSeed + #undef XXH3_createState + #undef XXH3_freeState + #undef XXH3_copyState + #undef XXH3_64bits_reset + #undef XXH3_64bits_reset_withSeed + #undef XXH3_64bits_reset_withSecret + #undef XXH3_64bits_update + #undef XXH3_64bits_digest + #undef XXH3_generateSecret +/* XXH3_128bits */ + #undef XXH128 + #undef XXH3_128bits + #undef XXH3_128bits_withSeed + #undef XXH3_128bits_withSecret + #undef XXH3_128bits_reset + #undef XXH3_128bits_reset_withSeed + #undef XXH3_128bits_reset_withSecret + #undef XXH3_128bits_update + #undef XXH3_128bits_digest + #undef XXH128_isEqual + #undef XXH128_cmp + #undef XXH128_canonicalFromHash + #undef XXH128_hashFromCanonical +/* Finally, free the namespace itself */ + #undef XXH_NAMESPACE + +/* employ the namespace for XXH_INLINE_ALL */ #define XXH_NAMESPACE XXH_INLINE_ /* - * Some identifiers (enums, type names) are not symbols, but they must - * still be renamed to avoid redeclaration. + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. * Alternative solution: do not redeclare them. - * However, this requires some #ifdefs, and is a more dispersed action. - * Meanwhile, renaming can be achieved in a single block + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. */ - #define XXH_IPREF(Id) XXH_INLINE_##Id + #define XXH_IPREF(Id) XXH_NAMESPACE##Id #define XXH_OK XXH_IPREF(XXH_OK) #define XXH_ERROR XXH_IPREF(XXH_ERROR) #define XXH_errorcode XXH_IPREF(XXH_errorcode) @@ -166,6 +220,12 @@ extern "C" { #ifndef XXHASH_H_5627135585666179 #define XXHASH_H_5627135585666179 1 + /*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + + */ /* specific declaration modes for Windows */ #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) #if defined(WIN32) && defined(_MSC_VER) && \ @@ -180,19 +240,24 @@ extern "C" { #endif #endif - /*! - * XXH_NAMESPACE, aka Namespace Emulation: - * - * If you want to include _and expose_ xxHash functions from within your own - * library, but also want to avoid symbol collisions with other libraries - * which may also include xxHash, you can use XXH_NAMESPACE to automatically - * prefix any public symbol from xxhash library with the value of - * XXH_NAMESPACE (therefore, avoid empty or numeric values). - * - * Note that no change is required within the calling program as long as it - * includes `xxhash.h`: Regular symbol names will be automatically translated - * by this header. - */ + #ifdef XXH_DOXYGEN + /*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries + * which may also include xxHash, you can use XXH_NAMESPACE to automatically + * prefix any public symbol from xxhash library with the value of + * XXH_NAMESPACE (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically + * translated by this header. + */ + #define XXH_NAMESPACE /* YOUR NAME HERE */ + #undef XXH_NAMESPACE + #endif + #ifdef XXH_NAMESPACE #define XXH_CAT(A, B) A##B #define XXH_NAME2(A, B) XXH_CAT(A, B) @@ -264,10 +329,19 @@ extern "C" { ***************************************/ #define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MINOR 8 - #define XXH_VERSION_RELEASE 0 + #define XXH_VERSION_RELEASE 1 #define XXH_VERSION_NUMBER \ (XXH_VERSION_MAJOR * 100 * 100 + XXH_VERSION_MINOR * 100 + \ XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is only useful when xxHash is compiled as a shared library, as it is + * independent of the version defined in the header. + * + * @return `XXH_VERSION_NUMBER` as of when the libray was compiled. + */ XXH_PUBLIC_API unsigned XXH_versionNumber(void); /* **************************** @@ -279,15 +353,24 @@ typedef enum { XXH_OK = 0, XXH_ERROR } XXH_errorcode; /*-********************************************************************** * 32-bit hash ************************************************************************/ - #if !defined(__VMS) && \ + #if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + + #elif !defined(__VMS) && \ (defined(__cplusplus) || \ (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) #include -typedef uint32_t XXH32_hash_t; +typedef uint32_t XXH32_hash_t; + #else #include #if UINT_MAX == 0xFFFFFFFFUL -typedef unsigned int XXH32_hash_t; +typedef unsigned int XXH32_hash_t; #else #if ULONG_MAX == 0xFFFFFFFFUL typedef unsigned long XXH32_hash_t; @@ -298,24 +381,52 @@ typedef unsigned long XXH32_hash_t; #endif /*! - * XXH32(): - * Calculate the 32-bit hash of sequence "length" bytes stored at memory - * address "input". The memory between input & input+length must be valid - * (allocated and read-accessible). "seed" can be used to alter the result - * predictably. Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher - * benchmark): 5.4 GB/s - * - * Note: XXH3 provides competitive speed for both 32-bit and 64-bit systems, - * and offers true 64/128 bit hash results. It provides a superior level of - * dispersion, and greatly reduces the risks of collisions. + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is considered rather weak by today's standards. + * The @ref xxh3_family provides competitive speed for both 32-bit and 64-bit + * systems, and offers true 64/128 bit hash results. It provides a superior + * level of dispersion, and greatly reduces the risks of collisions. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. */ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t length, XXH32_hash_t seed); -/******* Streaming *******/ - -/* - * Streaming functions generate the xxHash value from an incrememtal input. +/*! + * Streaming functions generate the xxHash value from an incremental input. * This method is slower than single-call functions, due to state management. * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. * @@ -336,19 +447,125 @@ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t length, * digest, and generate new hash values later on by invoking `XXH*_digest()`. * * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include + * #include + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. */ +typedef struct XXH32_state_s XXH32_state_t; -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ XXH_PUBLIC_API XXH32_state_t *XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr); -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t * dst_state, - const XXH32_state_t *src_state); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref + * XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t * dst_state, + const XXH32_state_t *src_state); +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *statePtr, const void *input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *statePtr); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *statePtr); /******* Canonical representation *******/ @@ -373,48 +590,158 @@ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *statePtr); * canonical format. */ +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ typedef struct { - unsigned char digest[4]; + unsigned char digest[4]; /*!< Hash bytes, big endian */ } XXH32_canonical_t; +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t *src); + #ifdef __has_attribute + #define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) + #else + #define XXH_HAS_ATTRIBUTE(x) 0 + #endif + + /* C-language Attributes are added in C23. */ + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && \ + defined(__has_c_attribute) + #define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) + #else + #define XXH_HAS_C_ATTRIBUTE(x) 0 + #endif + + #if defined(__cplusplus) && defined(__has_cpp_attribute) + #define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) + #else + #define XXH_HAS_CPP_ATTRIBUTE(x) 0 + #endif + + /* + Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' + attribute introduced in CPP17 and C23. CPP17 : + https://en.cppreference.com/w/cpp/language/attributes/fallthrough C23 : + https://en.cppreference.com/w/c/language/attributes/fallthrough + */ + #if XXH_HAS_C_ATTRIBUTE(x) + #define XXH_FALLTHROUGH [[fallthrough]] + #elif XXH_HAS_CPP_ATTRIBUTE(x) + #define XXH_FALLTHROUGH [[fallthrough]] + #elif XXH_HAS_ATTRIBUTE(__fallthrough__) + #define XXH_FALLTHROUGH __attribute__((fallthrough)) + #else + #define XXH_FALLTHROUGH + #endif + +/*! + * @} + * @ingroup public + * @{ + + */ + #ifndef XXH_NO_LONG_LONG /*-********************************************************************** * 64-bit hash ************************************************************************/ - #if !defined(__VMS) && \ + #if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; + #elif !defined(__VMS) && \ (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ (__STDC_VERSION__ >= 199901L) /* C99 */)) #include typedef uint64_t XXH64_hash_t; #else + #include + #if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL +/* LP64 ABI says uint64_t is unsigned long */ +typedef unsigned long XXH64_hash_t; + #else /* the following type must have a width of 64-bit */ typedef unsigned long long XXH64_hash_t; + #endif #endif /*! - * XXH64(): - * Returns the 64-bit hash of sequence of length @length stored at memory - * address @input. - * @seed can be used to alter the result predictably. + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. It provides a superior level of + * dispersion, and greatly reduces the risks of collisions. + */ + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. * * This function usually runs faster on 64-bit systems, but slower on 32-bit * systems (see benchmark). * - * Note: XXH3 provides competitive speed for both 32-bit and 64-bit systems, - * and offers true 64/128 bit hash results. It provides a superior level of - * dispersion, and greatly reduces the risks of collisions. + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. */ XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t length, XXH64_hash_t seed); /******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ XXH_PUBLIC_API XXH64_state_t *XXH64_createState(void); XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr); @@ -439,12 +766,15 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t *src); -/*-********************************************************************** - * XXH3 64-bit variant - ************************************************************************/ +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ -/* ************************************************************************ - * XXH3 is a new hash algorithm featuring: + * + * XXH3 is a more recent hash algorithm featuring: * - Improved speed for both small and large inputs * - True 64-bit and 128-bit outputs * - SIMD acceleration @@ -454,41 +784,38 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src); * * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html * - * In general, expect XXH3 to run about ~2x faster on large inputs and >3x - * faster on small ones compared to XXH64, though exact differences depend on - * the platform. - * - * The algorithm is portable: Like XXH32 and XXH64, it generates the same hash - * on all platforms. + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. * - * It benefits greatly from SIMD and 64-bit arithmetic, but does not require it. - * - * Almost all 32-bit and 64-bit targets that can run XXH32 smoothly can run - * XXH3 at competitive speeds, even if XXH64 runs slowly. Further details are - * explained in the implementation. + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. * * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, - * ZVector and scalar targets. This can be controlled with the XXH_VECTOR macro. + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. * * XXH3 offers 2 variants, _64bits and _128bits. - * When only 64 bits are needed, prefer calling the _64bits variant, as it - * reduces the amount of mixing, resulting in faster speed on small inputs. * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. * It's also generally simpler to manipulate a scalar return type than a struct. * - * The 128-bit version adds additional strength, but it is slightly slower. - * - * Return values of XXH3 and XXH128 are officially finalized starting - * with v0.8.0 and will no longer change in future versions. - * Avoid storing values from before that release in long-term storage. - * - * Results produced by v0.7.x are not comparable with results from v0.7.y. - * However, the API is completely stable, and it can safely be used for - * ephemeral data (local sessions). - * * The API supports one-shot hashing, streaming mode, and custom secrets. */ +/*-********************************************************************** + * XXH3 64-bit variant + ************************************************************************/ + /* XXH3_64bits(): * default 64-bit variant, using default secret and default seed of 0. * It's the fastest variant. */ @@ -504,20 +831,28 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *data, size_t len); XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void *data, size_t len, XXH64_hash_t seed); - /* - * XXH3_64bits_withSecret(): - * It's possible to provide any blob of bytes as a "secret" to generate the - * hash. This makes it more difficult for an external actor to prepare an - * intentional collision. The main condition is that secretSize *must* be - * large enough (>= XXH3_SECRET_SIZE_MIN). However, the quality of produced - * hash values depends on secret's entropy. Technically, the secret must - * look like a bunch of random bytes. Avoid "trivial" or structured data - * such as repeated sequences or a text document. Whenever unsure about the - * "randomness" of the blob of bytes, consider relabelling it as a "custom - * seed" instead, and employ "XXH3_generateSecret()" (see below) to generate - * a high entropy secret derived from the custom seed. + /*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). */ #define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the + * hash. This makes it more difficult for an external actor to prepare an + * intentional collision. The main condition is that secretSize *must* be large + * enough (>= XXH3_SECRET_SIZE_MIN). However, the quality of produced hash + * values depends on secret's entropy. Technically, the secret must look like a + * bunch of random bytes. Avoid "trivial" or structured data such as repeated + * sequences or a text document. Whenever unsure about the "randomness" of the + * blob of bytes, consider relabelling it as a "custom seed" instead, and employ + * "XXH3_generateSecret()" (see below) to generate a high entropy secret derived + * from the custom seed. + */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *data, size_t len, const void *secret, size_t secretSize); @@ -529,6 +864,12 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *data, size_t len, * As a consequence, streaming is slower than one-shot hashing. * For better performance, prefer one-shot functions whenever applicable. */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ typedef struct XXH3_state_s XXH3_state_t; XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void); XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr); @@ -572,10 +913,16 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *statePtr); * XXH3 128-bit variant ************************************************************************/ +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ typedef struct { - XXH64_hash_t low64; - XXH64_hash_t high64; + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ } XXH128_hash_t; @@ -649,6 +996,9 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src); #endif /* XXH_NO_LONG_LONG */ +/*! + * @} + */ #endif /* XXHASH_H_5627135585666179 */ #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) @@ -660,7 +1010,7 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src); * These declarations should only be used with static linking. * Never use them in association with dynamic linking! ***************************************************************************** - */ +*/ /* * These definitions are only present to allow static allocation @@ -668,41 +1018,72 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src); * Never **ever** access their members directly. */ +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ struct XXH32_state_s { - XXH32_hash_t total_len_32; - XXH32_hash_t large_len; - XXH32_hash_t v1; - XXH32_hash_t v2; - XXH32_hash_t v3; - XXH32_hash_t v4; - XXH32_hash_t mem32[4]; - XXH32_hash_t memsize; - XXH32_hash_t - reserved; /* never read nor write, might be removed in a future version */ + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref + total_len_32 overflow) */ + XXH32_hash_t v1; /*!< First accumulator lane */ + XXH32_hash_t v2; /*!< Second accumulator lane */ + XXH32_hash_t v3; /*!< Third accumulator lane */ + XXH32_hash_t v4; /*!< Fourth accumulator lane */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as + unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may + be removed. */ }; /* typedef'd to XXH32_state_t */ #ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ struct XXH64_state_s { - XXH64_hash_t total_len; - XXH64_hash_t v1; - XXH64_hash_t v2; - XXH64_hash_t v3; - XXH64_hash_t v4; - XXH64_hash_t mem64[4]; - XXH32_hash_t memsize; - XXH32_hash_t reserved32; /* required for padding anyway */ - XXH64_hash_t reserved64; /* never read nor write, might be removed in a future - version */ + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v1; /*!< First accumulator lane */ + XXH64_hash_t v2; /*!< Second accumulator lane */ + XXH64_hash_t v3; /*!< Third accumulator lane */ + XXH64_hash_t v4; /*!< Fourth accumulator lane */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as + unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it + may be removed. */ }; /* typedef'd to XXH64_state_t */ - #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11+ */ + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 \ + */ #include #define XXH_ALIGN(n) alignas(n) + #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ + /* In C++ alignas() is a keyword */ + #define XXH_ALIGN(n) alignas(n) #elif defined(__GNUC__) #define XXH_ALIGN(n) __attribute__((aligned(n))) #elif defined(_MSC_VER) @@ -713,39 +1094,94 @@ struct XXH64_state_s { /* Old GCC versions only accept the attribute after the type in structures. */ - #if !(defined(__STDC_VERSION__) && \ - (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + #if !(defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && !(defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ && defined(__GNUC__) #define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) #else #define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type #endif + /*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ #define XXH3_INTERNALBUFFER_SIZE 256 + + /*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ #define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ struct XXH3_state_s { XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); - /* used to store a custom secret generated from a seed */ + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref + * XXH64_state_s */ XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); - XXH32_hash_t bufferedSize; - XXH32_hash_t reserved32; - size_t nbStripesSoFar; - XXH64_hash_t totalLen; - size_t nbStripesPerBlock; - size_t secretLimit; - XXH64_hash_t seed; - XXH64_hash_t reserved64; - const unsigned char *extSecret; /* reference to external secret; - * if == NULL, use .customSecret instead */ + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t reserved32; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see + * XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char *extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ /* note: there may be some padding at the end due to alignment on 64 bytes */ }; /* typedef'd to XXH3_state_t */ #undef XXH_ALIGN_MEMBER - /* When the XXH3_state_t structure is merely emplaced on stack, + /*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, * it should be initialized with XXH3_INITSTATE() or a memset() * in case its first reset uses XXH3_NNbits_reset_withSeed(). * This init can be omitted if the first reset uses default or _withSecret @@ -802,7 +1238,6 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, XXH64_hash_t seed); #endif /* XXH_NO_LONG_LONG */ - #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) #define XXH_IMPLEMENTATION #endif @@ -844,81 +1279,183 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, /* ************************************* * Tuning parameters ***************************************/ + /*! - * XXH_FORCE_MEMORY_ACCESS: - * By default, access to unaligned memory is controlled by `memcpy()`, which - * is safe and portable. - * - * Unfortunately, on some target/compiler combinations, the generated assembly - * is sub-optimal. + * @defgroup tuning Tuning parameters + * @{ + * - * The below switch allow selection of a different access method - * in the search for improved performance. - * Method 0 (default): - * Use `memcpy()`. Safe and portable. Default. - * Method 1: - * `__attribute__((packed))` statement. It depends on compiler extensions - * and is therefore not portable. - * This method is safe if your compiler supports it, and *generally* as - * fast or faster than `memcpy`. - * Method 2: - * Direct access via cast. This method doesn't depend on the compiler but - * violates the C standard. - * It can generate buggy code on targets which do not support unaligned - * memory accesses. - * But in some circumstances, it's the only known way to get the most - * performance (example: GCC + ARMv6) - * Method 3: - * Byteshift. This can generate the best code on old compilers which don't - * inline small `memcpy()` calls, and it might also be faster on - * big-endian systems which lack a native byteswap instruction. See - * https://stackoverflow.com/a/32095106/646947 for details. Prefer these - * methods in priority order (0 > 1 > 2 > 3) + * Various macros to control xxHash's behavior. */ + #ifdef XXH_DOXYGEN + /*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 + * compiler. + */ + #define XXH_NO_LONG_LONG + #undef XXH_NO_LONG_LONG /* don't actually */ + /*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which + * is safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated + * assembly is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers + * will eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences + * an unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's + * the only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which + * don't inline small `memcpy()` calls, and it might also be faster on + * big-endian systems which lack a native byteswap instruction. However, + * some compilers will emit literal byteshifts even if the target supports + * unaligned access. + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may + * cause another to read garbage data or even crash. + * + * See https://stackoverflow.com/a/32095106/646947 for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ + #define XXH_FORCE_MEMORY_ACCESS 0 + /*! + * @def XXH_ACCEPT_NULL_INPUT_POINTER + * @brief Whether to add explicit `NULL` checks. + * + * If the input pointer is `NULL` and the length is non-zero, xxHash's + * default behavior is to dereference it, triggering a segfault. + * + * When this macro is enabled, xxHash actively checks the input for a null + * pointer. If it is, the result for null input pointers is the same as a + * zero-length input. + */ + #define XXH_ACCEPT_NULL_INPUT_POINTER 0 + /*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs + * (XXH32() and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally + * negligible, but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro + * to 0. Then the code will always use unaligned memory access. Align check + * is automatically disabled on x86, x64 & arm64, which are platforms known + * to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ + #define XXH_FORCE_ALIGN_CHECK 0 + + /*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all + * internal functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary + * which might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to + * performance, depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ + #define XXH_NO_INLINE_HINTS 0 + + /*! + * @def XXH_REROLL + * @brief Whether to reroll `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses an unrolled loop + * in the form of a switch statement. + * + * This is not always desirable, as it generates larger code, + * and depending on the architecture, may even be slower + * + * This is automatically defined with `-Os`/`-Oz` on GCC and Clang. + */ + #define XXH_REROLL 0 + + /*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use + * this. + */ + #define XXH_OLD_NAMES + #undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + #endif /* XXH_DOXYGEN */ +/*! + * @} + */ + #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command \ line for example */ - #if !defined(__clang__) && defined(__GNUC__) && \ - defined(__ARM_FEATURE_UNALIGNED) && defined(__ARM_ARCH) && \ - (__ARM_ARCH == 6) - #define XXH_FORCE_MEMORY_ACCESS 2 - #elif !defined(__clang__) && \ - ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ - (defined(__GNUC__) && (defined(__ARM_ARCH) && __ARM_ARCH >= 7))) + /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ + #if !defined(__clang__) && \ + ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + (defined(__GNUC__) && \ + ((defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ + (defined(__mips__) && (__mips <= 5 || __mips_isa_rev < 6) && \ + (!defined(__mips16) || defined(__mips_mips16e2)))))) #define XXH_FORCE_MEMORY_ACCESS 1 #endif #endif - /*! - * XXH_ACCEPT_NULL_INPUT_POINTER: - * If the input pointer is NULL, xxHash's default behavior is to dereference - * it, triggering a segfault. When this macro is enabled, xxHash actively - * checks the input for a null pointer. If it is, the result for null input - * pointers is the same as a zero-length input. - */ #ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ #define XXH_ACCEPT_NULL_INPUT_POINTER 0 #endif - /*! - * XXH_FORCE_ALIGN_CHECK: - * This is an important performance trick - * for architectures without decent unaligned memory access performance. - * It checks for input alignment, and when conditions are met, - * uses a "fast path" employing direct 32-bit/64-bit read, - * resulting in _dramatically faster_ read speed. - * - * The check costs one initial branch per hash, which is generally negligible, - * but not zero. Moreover, it's not useful to generate binary for an - * additional code path if memory access uses same instruction for both - * aligned and unaligned adresses. - * - * In these cases, the alignment check can be removed by setting this macro to - * 0. Then the code will always use unaligned memory access. Align check is - * automatically disabled on x86, x64 & arm64, which are platforms known to - * offer good unaligned memory accesses performance. - * - * This option does not affect XXH3 (only XXH32 and XXH64). - */ #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ #if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || \ defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ @@ -928,25 +1465,6 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, #endif #endif - /*! - * XXH_NO_INLINE_HINTS: - * - * By default, xxHash tries to force the compiler to inline almost all - * internal functions. - * - * This can usually improve performance due to reduced jumping and improved - * constant folding, but significantly increases the size of the binary which - * might not be favorable. - * - * Additionally, sometimes the forced inlining can be detrimental to - * performance, depending on the architecture. - * - * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the - * compiler full control on whether to inline or not. - * - * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using - * -fno-inline with GCC or Clang, this will automatically be defined. - */ #ifndef XXH_NO_INLINE_HINTS #if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ || defined(__NO_INLINE__) /* -O0, -fno-inline */ @@ -956,44 +1474,57 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, #endif #endif - /*! - * XXH_REROLL: - * Whether to reroll XXH32_finalize, and XXH64_finalize, - * instead of using an unrolled jump table/if statement loop. - * - * This is automatically defined on -Os/-Oz on GCC and Clang. - */ #ifndef XXH_REROLL - #if defined(__OPTIMIZE_SIZE__) + #if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ || \ + (defined(__GNUC__) && !defined(__clang__)) + /* The if/then loop is preferable to switch/case on gcc (on x64) */ #define XXH_REROLL 1 #else #define XXH_REROLL 0 #endif #endif + /*! + * @defgroup impl Implementation + * @{ + + */ + /* ************************************* * Includes & Memory related functions ***************************************/ - /*! + /* * Modify the local functions below should you wish to use * different memory routines for malloc() and free() */ #include +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ static void *XXH_malloc(size_t s) { return malloc(s); } +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ static void XXH_free(void *p) { free(p); } - /*! and for memcpy() */ #include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ static void *XXH_memcpy(void *dest, const void *src, size_t size) { return memcpy(dest, src, size); @@ -1037,7 +1568,11 @@ static void *XXH_memcpy(void *dest, const void *src, size_t size) { /* ************************************* * Debug ***************************************/ - /* + /*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * * XXH_DEBUGLEVEL is expected to be defined externally, typically via the * compiler's command line options. The value must be a number. */ @@ -1057,12 +1592,58 @@ static void *XXH_memcpy(void *dest, const void *src, size_t size) { #endif /* note: use after variable declarations */ - #define XXH_STATIC_ASSERT(c) \ - do { \ - \ - enum { XXH_sa = 1 / (int)(!!(c)) }; \ - \ - } while (0) + #ifndef XXH_STATIC_ASSERT + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ + #include + #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ + do { \ + \ + static_assert((c), m); \ + \ + } while (0) + #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ + #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ + do { \ + \ + static_assert((c), m); \ + \ + } while (0) + #else + #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ + do { \ + \ + struct xxh_sa { \ + \ + char x[(c) ? 1 : -1]; \ + \ + }; \ + \ + } while (0) + #endif + #define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c), #c) + #endif + + /*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ + #ifdef __GNUC__ + #define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r"(var)) + #else + #define XXH_COMPILER_GUARD(var) ((void)0) + #endif /* ************************************* * Basic Types @@ -1085,6 +1666,56 @@ typedef XXH32_hash_t xxh_u32; /* *** Memory access *** */ +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) /* * Manual byteshift. Best for old compilers which don't inline memcpy. @@ -1146,16 +1777,23 @@ static xxh_u32 XXH_read32(const void *memPtr) { #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ -/* *** Endianess *** */ -typedef enum { XXH_bigEndian = 0, XXH_littleEndian = 1 } XXH_endianess; + /* *** Endianness *** */ /*! - * XXH_CPU_LITTLE_ENDIAN: + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * * Defined to 1 if the target is little endian, or 0 if it is big endian. * It can be defined externally, for example on the compiler command line. * - * If it is not defined, a runtime check (which is usually constant folded) - * is used instead. + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. */ #ifndef XXH_CPU_LITTLE_ENDIAN /* @@ -1170,8 +1808,11 @@ typedef enum { XXH_bigEndian = 0, XXH_littleEndian = 1 } XXH_endianess; (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #define XXH_CPU_LITTLE_ENDIAN 0 #else -/* - * runtime test, presumed to simplify to a constant by compiler +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. */ static int XXH_isLittleEndian(void) { @@ -1189,7 +1830,7 @@ static int XXH_isLittleEndian(void) { return one.c[0]; } - +\ #define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() #endif #endif @@ -1205,6 +1846,19 @@ static int XXH_isLittleEndian(void) { #define XXH_HAS_BUILTIN(x) 0 #endif + /*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ #if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) && \ XXH_HAS_BUILTIN(__builtin_rotateleft64) #define XXH_rotl32 __builtin_rotateleft32 @@ -1219,6 +1873,14 @@ static int XXH_isLittleEndian(void) { #define XXH_rotl64(x, r) (((x) << (r)) | ((x) >> (64 - (r)))) #endif + /*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ #if defined(_MSC_VER) /* Visual Studio */ #define XXH_swap32 _byteswap_ulong #elif XXH_GCC_VERSION >= 403 @@ -1236,7 +1898,17 @@ static xxh_u32 XXH_swap32(xxh_u32 x) { /* *************************** * Memory reads *****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ + +} XXH_alignment; /* * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. @@ -1295,6 +1967,7 @@ XXH_FORCE_INLINE xxh_u32 XXH_readLE32_align(const void * ptr, /* ************************************* * Misc ***************************************/ +/*! @ingroup public */ XXH_PUBLIC_API unsigned XXH_versionNumber(void) { return XXH_VERSION_NUMBER; @@ -1304,16 +1977,19 @@ XXH_PUBLIC_API unsigned XXH_versionNumber(void) { /* ******************************************************************* * 32-bit hash functions *********************************************************************/ -static const xxh_u32 XXH_PRIME32_1 = - 0x9E3779B1U; /* 0b10011110001101110111100110110001 */ -static const xxh_u32 XXH_PRIME32_2 = - 0x85EBCA77U; /* 0b10000101111010111100101001110111 */ -static const xxh_u32 XXH_PRIME32_3 = - 0xC2B2AE3DU; /* 0b11000010101100101010111000111101 */ -static const xxh_u32 XXH_PRIME32_4 = - 0x27D4EB2FU; /* 0b00100111110101001110101100101111 */ -static const xxh_u32 XXH_PRIME32_5 = - 0x165667B1U; /* 0b00010110010101100110011110110001 */ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + + */ +/* #define instead of static const, to be used as initializers */ + #define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ + #define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ + #define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ + #define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ + #define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ #ifdef XXH_OLD_NAMES #define PRIME32_1 XXH_PRIME32_1 @@ -1323,19 +1999,29 @@ static const xxh_u32 XXH_PRIME32_5 = #define PRIME32_5 XXH_PRIME32_5 #endif +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { acc += input * XXH_PRIME32_2; acc = XXH_rotl32(acc, 13); acc *= XXH_PRIME32_1; - #if defined(__GNUC__) && defined(__SSE4_1__) && \ + #if (defined(__SSE4_1__) || defined(__aarch64__)) && \ !defined(XXH_ENABLE_AUTOVECTORIZE) /* * UGLY HACK: - * This inline assembly hack forces acc into a normal register. This is the - * only thing that prevents GCC and Clang from autovectorizing the XXH32 - * loop (pragmas and attributes don't work for some resason) without globally - * disabling SSE4.1. + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. * * The reason we want to avoid vectorization is because despite working on * 4 integers at a time, there are multiple factors slowing XXH32 down on @@ -1360,28 +2046,26 @@ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { * can load data, while v3 can multiply. SSE forces them to operate * together. * - * How this hack works: - * __asm__("" // Declare an assembly block but don't declare any - * instructions : // However, as an Input/Output Operand, - * "+r" // constrain a read/write operand (+) as a general purpose - * register (r). (acc) // and set acc as the operand - * ); - * - * Because of the 'r', the compiler has promised that seed will be in a - * general purpose register and the '+' says that it will be 'read/write', - * so it has to assume it has changed. It is like volatile without all the - * loads and stores. - * - * Since the argument has to be in a normal register (not an SSE register), - * each time XXH32_round is called, it is impossible to vectorize. + * This is also enabled on AArch64, as Clang autovectorizes it incorrectly + * and it is pointless writing a NEON implementation that is basically the + * same speed as scalar for XXH32. */ - __asm__("" : "+r"(acc)); + XXH_COMPILER_GUARD(acc); #endif return acc; } -/* mix all bits */ +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ static xxh_u32 XXH32_avalanche(xxh_u32 h32) { h32 ^= h32 >> 15; @@ -1395,11 +2079,23 @@ static xxh_u32 XXH32_avalanche(xxh_u32 h32) { #define XXH_get32bits(p) XXH_readLE32_align(p, align) +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, XXH_alignment align) { - - /* dummy comment */ - +\ #define XXH_PROCESS1 \ do { \ \ @@ -1443,20 +2139,20 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, case 12: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 8: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 4: XXH_PROCESS4; return XXH32_avalanche(h32); case 13: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 9: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 5: XXH_PROCESS4; XXH_PROCESS1; @@ -1464,10 +2160,10 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, case 14: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 10: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 6: XXH_PROCESS4; XXH_PROCESS1; @@ -1476,22 +2172,22 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, case 15: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 11: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 7: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 3: XXH_PROCESS1; - /* fallthrough */ + XXH_FALLTHROUGH; case 2: XXH_PROCESS1; - /* fallthrough */ + XXH_FALLTHROUGH; case 1: XXH_PROCESS1; - /* fallthrough */ + XXH_FALLTHROUGH; case 0: return XXH32_avalanche(h32); @@ -1512,10 +2208,18 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, #undef XXH_PROCESS4 #endif +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input, len, seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ XXH_FORCE_INLINE xxh_u32 XXH32_endian_align(const xxh_u8 *input, size_t len, xxh_u32 seed, XXH_alignment align) { - const xxh_u8 *bEnd = input + len; + const xxh_u8 *bEnd = input ? input + len : NULL; xxh_u32 h32; #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ @@ -1565,6 +2269,7 @@ XXH_FORCE_INLINE xxh_u32 XXH32_endian_align(const xxh_u8 *input, size_t len, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, XXH32_hash_t seed) { @@ -1574,9 +2279,7 @@ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, XXH32_reset(&state, seed); XXH32_update(&state, (const xxh_u8*)input, len); return XXH32_digest(&state); - #else - if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 3) == @@ -1593,13 +2296,16 @@ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, } /******* Hash streaming *******/ - +/*! + * @ingroup xxh32_family + */ XXH_PUBLIC_API XXH32_state_t *XXH32_createState(void) { return (XXH32_state_t *)XXH_malloc(sizeof(XXH32_state_t)); } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr) { XXH_free(statePtr); @@ -1607,6 +2313,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr) { } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t * dstState, const XXH32_state_t *srcState) { @@ -1614,6 +2321,7 @@ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t * dstState, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, XXH32_hash_t seed) { @@ -1630,6 +2338,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, const void *input, size_t len) { @@ -1719,6 +2428,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *state) { xxh_u32 h32; @@ -1743,7 +2453,8 @@ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *state) { /******* Canonical representation *******/ -/* +/*! + * @ingroup xxh32_family * The default return values from XXH functions are unsigned 32 and 64 bit * integers. * @@ -1765,6 +2476,7 @@ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t *src) { @@ -1777,7 +2489,12 @@ XXH32_hashFromCanonical(const XXH32_canonical_t *src) { /* ******************************************************************* * 64-bit hash functions *********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ /******* Memory access *******/ typedef XXH64_hash_t xxh_u64; @@ -1786,40 +2503,6 @@ typedef XXH64_hash_t xxh_u64; #define U64 xxh_u64 #endif - /*! - * XXH_REROLL_XXH64: - * Whether to reroll the XXH64_finalize() loop. - * - * Just like XXH32, we can unroll the XXH64_finalize() loop. This can be a - * performance gain on 64-bit hosts, as only one jump is required. - * - * However, on 32-bit hosts, because arithmetic needs to be done with two - * 32-bit registers, and 64-bit arithmetic needs to be simulated, it isn't - * beneficial to unroll. The code becomes ridiculously large (the largest - * function in the binary on i386!), and rerolling it saves anywhere from - * 3kB to 20kB. It is also slightly faster because it fits into cache better - * and is more likely to be inlined by the compiler. - * - * If XXH_REROLL is defined, this is ignored and the loop is always - * rerolled. - */ - #ifndef XXH_REROLL_XXH64 - #if (defined(__ILP32__) || \ - defined(_ILP32)) /* ILP32 is often defined on 32-bit GCC family */ \ - || !(defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) /* x86-64 */ \ - || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__arm64__) /* aarch64 */ \ - || defined(__PPC64__) || defined(__PPC64LE__) || \ - defined(__ppc64__) || defined(__powerpc64__) /* ppc64 */ \ - || defined(__mips64__) || defined(__mips64)) /* mips64 */ \ - || (!defined(SIZE_MAX) || SIZE_MAX < ULLONG_MAX) /* check limits */ - #define XXH_REROLL_XXH64 1 - #else - #define XXH_REROLL_XXH64 0 - #endif - #endif /* !defined(XXH_REROLL_XXH64) */ - #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) /* * Manual byteshift. Best for old compilers which don't inline memcpy. @@ -1950,23 +2633,35 @@ XXH_FORCE_INLINE xxh_u64 XXH_readLE64_align(const void * ptr, } -/******* xxh64 *******/ + /******* xxh64 *******/ + /*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ -static const xxh_u64 XXH_PRIME64_1 = - 0x9E3779B185EBCA87ULL; /* 0b1001111000110111011110011011000110000101111010111100101010000111 - */ -static const xxh_u64 XXH_PRIME64_2 = - 0xC2B2AE3D27D4EB4FULL; /* 0b1100001010110010101011100011110100100111110101001110101101001111 - */ -static const xxh_u64 XXH_PRIME64_3 = - 0x165667B19E3779F9ULL; /* 0b0001011001010110011001111011000110011110001101110111100111111001 - */ -static const xxh_u64 XXH_PRIME64_4 = - 0x85EBCA77C2B2AE63ULL; /* 0b1000010111101011110010100111011111000010101100101010111001100011 - */ -static const xxh_u64 XXH_PRIME64_5 = - 0x27D4EB2F165667C5ULL; /* 0b0010011111010100111010110010111100010110010101100110011111000101 - */ + */ + /* #define rather that static const, to be used as initializers */ + #define XXH_PRIME64_1 \ + 0x9E3779B185EBCA87ULL /*!< \ + 0b1001111000110111011110011011000110000101111010111100101010000111 \ + */ + #define XXH_PRIME64_2 \ + 0xC2B2AE3D27D4EB4FULL /*!< \ + 0b1100001010110010101011100011110100100111110101001110101101001111 \ + */ + #define XXH_PRIME64_3 \ + 0x165667B19E3779F9ULL /*!< \ + 0b0001011001010110011001111011000110011110001101110111100111111001 \ + */ + #define XXH_PRIME64_4 \ + 0x85EBCA77C2B2AE63ULL /*!< \ + 0b1000010111101011110010100111011111000010101100101010111001100011 \ + */ + #define XXH_PRIME64_5 \ + 0x27D4EB2F165667C5ULL /*!< \ + 0b0010011111010100111010110010111100010110010101100110011111000101 \ + */ #ifdef XXH_OLD_NAMES #define PRIME64_1 XXH_PRIME64_1 @@ -2010,185 +2705,35 @@ static xxh_u64 XXH64_avalanche(xxh_u64 h64) { static xxh_u64 XXH64_finalize(xxh_u64 h64, const xxh_u8 *ptr, size_t len, XXH_alignment align) { - /* dummy comment */ - - #define XXH_PROCESS1_64 \ - do { \ - \ - h64 ^= (*ptr++) * XXH_PRIME64_5; \ - h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; \ - \ - } while (0) - - #define XXH_PROCESS4_64 \ - do { \ - \ - h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; \ - ptr += 4; \ - h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; \ - \ - } while (0) - - #define XXH_PROCESS8_64 \ - do { \ - \ - xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); \ - ptr += 8; \ - h64 ^= k1; \ - h64 = XXH_rotl64(h64, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; \ - \ - } while (0) - - /* Rerolled version for 32-bit targets is faster and much smaller. */ - if (XXH_REROLL || XXH_REROLL_XXH64) { - - len &= 31; - while (len >= 8) { - - XXH_PROCESS8_64; - len -= 8; - - } - - if (len >= 4) { - - XXH_PROCESS4_64; - len -= 4; - - } - - while (len > 0) { + len &= 31; + while (len >= 8) { - XXH_PROCESS1_64; - --len; + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + h64 ^= k1; + h64 = XXH_rotl64(h64, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; - } + } - return XXH64_avalanche(h64); + if (len >= 4) { - } else { + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; - switch (len & 31) { + } - case 24: - XXH_PROCESS8_64; - /* fallthrough */ - case 16: - XXH_PROCESS8_64; - /* fallthrough */ - case 8: - XXH_PROCESS8_64; - return XXH64_avalanche(h64); - - case 28: - XXH_PROCESS8_64; - /* fallthrough */ - case 20: - XXH_PROCESS8_64; - /* fallthrough */ - case 12: - XXH_PROCESS8_64; - /* fallthrough */ - case 4: - XXH_PROCESS4_64; - return XXH64_avalanche(h64); - - case 25: - XXH_PROCESS8_64; - /* fallthrough */ - case 17: - XXH_PROCESS8_64; - /* fallthrough */ - case 9: - XXH_PROCESS8_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 29: - XXH_PROCESS8_64; - /* fallthrough */ - case 21: - XXH_PROCESS8_64; - /* fallthrough */ - case 13: - XXH_PROCESS8_64; - /* fallthrough */ - case 5: - XXH_PROCESS4_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 26: - XXH_PROCESS8_64; - /* fallthrough */ - case 18: - XXH_PROCESS8_64; - /* fallthrough */ - case 10: - XXH_PROCESS8_64; - XXH_PROCESS1_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 30: - XXH_PROCESS8_64; - /* fallthrough */ - case 22: - XXH_PROCESS8_64; - /* fallthrough */ - case 14: - XXH_PROCESS8_64; - /* fallthrough */ - case 6: - XXH_PROCESS4_64; - XXH_PROCESS1_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 27: - XXH_PROCESS8_64; - /* fallthrough */ - case 19: - XXH_PROCESS8_64; - /* fallthrough */ - case 11: - XXH_PROCESS8_64; - XXH_PROCESS1_64; - XXH_PROCESS1_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 31: - XXH_PROCESS8_64; - /* fallthrough */ - case 23: - XXH_PROCESS8_64; - /* fallthrough */ - case 15: - XXH_PROCESS8_64; - /* fallthrough */ - case 7: - XXH_PROCESS4_64; - /* fallthrough */ - case 3: - XXH_PROCESS1_64; - /* fallthrough */ - case 2: - XXH_PROCESS1_64; - /* fallthrough */ - case 1: - XXH_PROCESS1_64; - /* fallthrough */ - case 0: - return XXH64_avalanche(h64); + while (len > 0) { - } + h64 ^= (*ptr++) * XXH_PRIME64_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + --len; } - /* impossible to reach */ - XXH_ASSERT(0); - return 0; /* unreachable, but some compilers complain without it */ + return XXH64_avalanche(h64); } @@ -2205,7 +2750,7 @@ static xxh_u64 XXH64_finalize(xxh_u64 h64, const xxh_u8 *ptr, size_t len, XXH_FORCE_INLINE xxh_u64 XXH64_endian_align(const xxh_u8 *input, size_t len, xxh_u64 seed, XXH_alignment align) { - const xxh_u8 *bEnd = input + len; + const xxh_u8 *bEnd = input ? input + len : NULL; xxh_u64 h64; #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ @@ -2259,6 +2804,7 @@ XXH_FORCE_INLINE xxh_u64 XXH64_endian_align(const xxh_u8 *input, size_t len, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t len, XXH64_hash_t seed) { @@ -2268,9 +2814,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t len, XXH64_reset(&state, seed); XXH64_update(&state, (const xxh_u8*)input, len); return XXH64_digest(&state); - #else - if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 7) == @@ -2289,12 +2833,14 @@ XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t len, /******* Hash Streaming *******/ +/*! @ingroup xxh64_family*/ XXH_PUBLIC_API XXH64_state_t *XXH64_createState(void) { return (XXH64_state_t *)XXH_malloc(sizeof(XXH64_state_t)); } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr) { XXH_free(statePtr); @@ -2302,6 +2848,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr) { } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t * dstState, const XXH64_state_t *srcState) { @@ -2309,6 +2856,7 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t * dstState, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t *statePtr, XXH64_hash_t seed) { @@ -2325,6 +2873,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t *statePtr, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *state, const void *input, size_t len) { @@ -2403,6 +2952,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *state, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t *state) { xxh_u64 h64; @@ -2436,6 +2986,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t *state) { /******* Canonical representation *******/ +/*! @ingroup xxh64_family */ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, XXH64_hash_t hash) { @@ -2445,6 +2996,7 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t *src) { @@ -2452,380 +3004,452 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src) { } - /* ********************************************************************* - * XXH3 - * New generation hash designed for speed on small keys and vectorization - ************************************************************************ */ + #ifndef XXH_NO_XXH3 - /* === Compiler specifics === */ + /* ********************************************************************* + * XXH3 + * New generation hash designed for speed on small keys and vectorization + ************************************************************************ */ + /*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ - #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ - #define XXH_RESTRICT restrict - #else - /* Note: it might be useful to define __restrict or __restrict__ for some - * C++ compilers */ - #define XXH_RESTRICT /* disable */ - #endif + */ - #if (defined(__GNUC__) && (__GNUC__ >= 3)) || \ - (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || \ - defined(__clang__) - #define XXH_likely(x) __builtin_expect(x, 1) - #define XXH_unlikely(x) __builtin_expect(x, 0) - #else - #define XXH_likely(x) (x) - #define XXH_unlikely(x) (x) - #endif + /* === Compiler specifics === */ - #if defined(__GNUC__) - #if defined(__AVX2__) - #include - #elif defined(__SSE2__) - #include - #elif defined(__ARM_NEON__) || defined(__ARM_NEON) - #define inline __inline__ /* circumvent a clang bug */ - #include - #undef inline + #if ((defined(sun) || defined(__sun)) && \ + __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested \ + with GCC 5.5 */ + #define XXH_RESTRICT /* disable */ + #elif defined(__STDC_VERSION__) && \ + __STDC_VERSION__ >= 199901L /* >= C99 */ + #define XXH_RESTRICT restrict + #else + /* Note: it might be useful to define __restrict or __restrict__ for + * some C++ compilers */ + #define XXH_RESTRICT /* disable */ #endif - #elif defined(_MSC_VER) - #include - #endif - /* - * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while - * remaining a true 64-bit/128-bit hash function. - * - * This is done by prioritizing a subset of 64-bit operations that can be - * emulated without too many steps on the average 32-bit machine. - * - * For example, these two lines seem similar, and run equally fast on - * 64-bit: - * - * xxh_u64 x; - * x ^= (x >> 47); // good - * x ^= (x >> 13); // bad - * - * However, to a 32-bit machine, there is a major difference. - * - * x ^= (x >> 47) looks like this: - * - * x.lo ^= (x.hi >> (47 - 32)); - * - * while x ^= (x >> 13) looks like this: - * - * // note: funnel shifts are not usually cheap. - * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); - * x.hi ^= (x.hi >> 13); - * - * The first one is significantly faster than the second, simply because the - * shift is larger than 32. This means: - * - All the bits we need are in the upper 32 bits, so we can ignore the - * lower 32 bits in the shift. - * - The shift result will always fit in the lower 32 bits, and therefore, - * we can ignore the upper 32 bits in the xor. - * - * Thanks to this optimization, XXH3 only requires these features to be - * efficient: - * - * - Usable unaligned access - * - A 32-bit or 64-bit ALU - * - If 32-bit, a decent ADC instruction - * - A 32 or 64-bit multiply with a 64-bit result - * - For the 128-bit variant, a decent byteswap helps short inputs. - * - * The first two are already required by XXH32, and almost all 32-bit and - * 64-bit platforms which can run XXH32 can run XXH3 efficiently. - * - * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one - * notable exception. - * - * First of all, Thumb-1 lacks support for the UMULL instruction which - * performs the important long multiply. This means numerous __aeabi_lmul - * calls. - * - * Second of all, the 8 functional registers are just not enough. - * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic - * need Lo registers, and this shuffling results in thousands more MOVs than - * A32. - * - * A32 and T32 don't have this limitation. They can access all 14 registers, - * do a 32->64 multiply with UMULL, and the flexible operand allowing free - * shifts is helpful, too. - * - * Therefore, we do a quick sanity check. - * - * If compiling Thumb-1 for a target which supports ARM instructions, we - * will emit a warning, as it is not a "sane" platform to compile for. - * - * Usually, if this happens, it is because of an accident and you probably - * need to specify -march, as you likely meant to compile for a newer - * architecture. - * - * Credit: large sections of the vectorial and asm source code paths - * have been contributed by @easyaspi314 - */ - #if defined(__thumb__) && !defined(__thumb2__) && \ - defined(__ARM_ARCH_ISA_ARM) - #warning "XXH3 is highly inefficient without ARM or Thumb-2." - #endif + #if (defined(__GNUC__) && (__GNUC__ >= 3)) || \ + (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || \ + defined(__clang__) + #define XXH_likely(x) __builtin_expect(x, 1) + #define XXH_unlikely(x) __builtin_expect(x, 0) + #else + #define XXH_likely(x) (x) + #define XXH_unlikely(x) (x) + #endif + + #if defined(__GNUC__) + #if defined(__AVX2__) + #include + #elif defined(__SSE2__) + #include + #elif defined(__ARM_NEON__) || defined(__ARM_NEON) + #define inline __inline__ /* circumvent a clang bug */ + #include + #undef inline + #endif + #elif defined(_MSC_VER) + #include + #endif + + /* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on + * 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because + * the shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the + * lower 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and + * therefore, we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be + * efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and + * 64-bit platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is + * one notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic + * need Lo registers, and this shuffling results in thousands more MOVs + * than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 + * registers, do a 32->64 multiply with UMULL, and the flexible operand + * allowing free shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we + * will emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably + * need to specify -march, as you likely meant to compile for a newer + * architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ + #if defined(__thumb__) && !defined(__thumb2__) && \ + defined(__ARM_ARCH_ISA_ARM) + #warning "XXH3 is highly inefficient without ARM or Thumb-2." + #endif /* ========================================== * Vectorization detection * ========================================== */ - #define XXH_SCALAR 0 /* Portable scalar version */ - #define XXH_SSE2 1 /* SSE2 for Pentium 4 and all x86_64 */ - #define XXH_AVX2 2 /* AVX2 for Haswell and Bulldozer */ - #define XXH_AVX512 3 /* AVX512 for Skylake and Icelake */ - #define XXH_NEON 4 /* NEON for most ARMv7-A and all AArch64 */ - #define XXH_VSX 5 /* VSX and ZVector for POWER8/z13 */ - - #ifndef XXH_VECTOR /* can be defined on command line */ - #if defined(__AVX512F__) - #define XXH_VECTOR XXH_AVX512 - #elif defined(__AVX2__) - #define XXH_VECTOR XXH_AVX2 - #elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) - #define XXH_VECTOR XXH_SSE2 - #elif defined(__GNUC__) /* msvc support maybe later */ \ - && (defined(__ARM_NEON__) || defined(__ARM_NEON)) && \ - (defined(__LITTLE_ENDIAN__) /* We only support little endian NEON */ \ - || (defined(__BYTE_ORDER__) && \ - __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) - #define XXH_VECTOR XXH_NEON - #elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) || \ - (defined(__s390x__) && defined(__VEC__)) && \ - defined(__GNUC__) /* TODO: IBM XL */ - #define XXH_VECTOR XXH_VSX - #else + + #ifdef XXH_DOXYGEN + /*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the + * best implementation. + */ #define XXH_VECTOR XXH_SCALAR - #endif - #endif +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ - /* - * Controls the alignment of the accumulator, - * for compatibility with aligned vector loads, which are usually faster. - */ - #ifndef XXH_ACC_ALIGN - #if defined(XXH_X86DISPATCH) - #define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ - #elif XXH_VECTOR == XXH_SCALAR /* scalar */ +}; + + /*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment reqired for said + * vector type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ #define XXH_ACC_ALIGN 8 - #elif XXH_VECTOR == XXH_SSE2 /* sse2 */ - #define XXH_ACC_ALIGN 16 - #elif XXH_VECTOR == XXH_AVX2 /* avx2 */ - #define XXH_ACC_ALIGN 32 - #elif XXH_VECTOR == XXH_NEON /* neon */ - #define XXH_ACC_ALIGN 16 - #elif XXH_VECTOR == XXH_VSX /* vsx */ - #define XXH_ACC_ALIGN 16 - #elif XXH_VECTOR == XXH_AVX512 /* avx512 */ - #define XXH_ACC_ALIGN 64 #endif - #endif - #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 || \ - XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 - #define XXH_SEC_ALIGN XXH_ACC_ALIGN - #else - #define XXH_SEC_ALIGN 8 - #endif + /* Actual definition */ + #ifndef XXH_DOXYGEN + #define XXH_SCALAR 0 + #define XXH_SSE2 1 + #define XXH_AVX2 2 + #define XXH_AVX512 3 + #define XXH_NEON 4 + #define XXH_VSX 5 + #endif + + #ifndef XXH_VECTOR /* can be defined on command line */ + #if defined(__AVX512F__) + #define XXH_VECTOR XXH_AVX512 + #elif defined(__AVX2__) + #define XXH_VECTOR XXH_AVX2 + #elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) + #define XXH_VECTOR XXH_SSE2 + #elif defined(__GNUC__) /* msvc support maybe later */ \ + && (defined(__ARM_NEON__) || defined(__ARM_NEON)) && \ + (defined( \ + __LITTLE_ENDIAN__) /* We only support little endian NEON */ \ + || (defined(__BYTE_ORDER__) && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + #define XXH_VECTOR XXH_NEON + #elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) || \ + (defined(__s390x__) && defined(__VEC__)) && \ + defined(__GNUC__) /* TODO: IBM XL */ + #define XXH_VECTOR XXH_VSX + #else + #define XXH_VECTOR XXH_SCALAR + #endif + #endif + + /* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ + #ifndef XXH_ACC_ALIGN + #if defined(XXH_X86DISPATCH) + #define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ + #elif XXH_VECTOR == XXH_SCALAR /* scalar */ + #define XXH_ACC_ALIGN 8 + #elif XXH_VECTOR == XXH_SSE2 /* sse2 */ + #define XXH_ACC_ALIGN 16 + #elif XXH_VECTOR == XXH_AVX2 /* avx2 */ + #define XXH_ACC_ALIGN 32 + #elif XXH_VECTOR == XXH_NEON /* neon */ + #define XXH_ACC_ALIGN 16 + #elif XXH_VECTOR == XXH_VSX /* vsx */ + #define XXH_ACC_ALIGN 16 + #elif XXH_VECTOR == XXH_AVX512 /* avx512 */ + #define XXH_ACC_ALIGN 64 + #endif + #endif - /* - * UGLY HACK: - * GCC usually generates the best code with -O3 for xxHash. - * - * However, when targeting AVX2, it is overzealous in its unrolling - * resulting in code roughly 3/4 the speed of Clang. - * - * There are other issues, such as GCC splitting _mm256_loadu_si256 into - * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which - * only applies to Sandy and Ivy Bridge... which don't even support AVX2. - * - * That is why when compiling the AVX2 version, it is recommended to use - * either -O2 -mavx2 -march=haswell or -O2 -mavx2 - * -mno-avx256-split-unaligned-load for decent performance, or to use Clang - * instead. - * - * Fortunately, we can control the first one with a pragma that forces GCC - * into -O2, but the other one we can't control without "failed to inline - * always inline function due to target mismatch" warnings. - */ - #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ - && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__OPTIMIZE__) && \ - !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ - #pragma GCC push_options - #pragma GCC optimize("-O2") - #endif + #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 || \ + XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 + #define XXH_SEC_ALIGN XXH_ACC_ALIGN + #else + #define XXH_SEC_ALIGN 8 + #endif - #if XXH_VECTOR == XXH_NEON /* - * NEON's setup for vmlal_u32 is a little more complicated than it is on - * SSE2, AVX2, and VSX. - * - * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an - * upcast. - * - * To do the same operation, the 128-bit 'Q' register needs to be split - * into two 64-bit 'D' registers, performing this operation:: - * - * [ a | b ] | - * '---------. .--------' | | x | - * | .---------' '--------. | - * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] - * - * Due to significant changes in aarch64, the fastest method for aarch64 - * is completely different than the fastest method for ARMv7-A. - * - * ARMv7-A treats D registers as unions overlaying Q registers, so - * modifying D11 will modify the high half of Q5. This is similar to how - * modifying AH will only affect bits 8-15 of AX on x86. - * - * VZIP takes two registers, and puts even lanes in one register and odd - * lanes in the other. - * - * On ARMv7-A, this strangely modifies both parameters in place instead of - * taking the usual 3-operand form. - * - * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on - * the lower and upper halves of the Q register to end up with the high - * and low halves where we want - all in one instruction. - * - * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], - * d11[1] } - * - * Unfortunately we need inline assembly for this: Instructions modifying - * two registers at once is not possible in GCC or Clang's IR, and they - * have to create a copy. - * - * aarch64 requires a different approach. - * - * In order to make it easier to write a decent compiler for aarch64, many - * quirks were removed, such as conditional execution. - * - * NEON was also affected by this. - * - * aarch64 cannot access the high bits of a Q-form register, and writes to - * a D-form register zero the high bits, similar to how writes to W-form - * scalar registers (or DWORD registers on x86_64) work. - * - * The formerly free vget_high intrinsics now require a vext (with a few - * exceptions) - * - * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the - * equivalent of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to - * only modify one operand. - * - * The equivalent of the VZIP.32 on the lower and upper halves would be - * this mess: + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. * - * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] - * } zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } zip2 v0.2s, - * v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * However, when targeting AVX2, it is overzealous in its unrolling + * resulting in code roughly 3/4 the speed of Clang. * - * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 - * (SHRN): + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization + * which only applies to Sandy and Ivy Bridge... which don't even support + * AVX2. * - * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); - * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * That is why when compiling the AVX2 version, it is recommended to use + * either -O2 -mavx2 -march=haswell or -O2 -mavx2 + * -mno-avx256-split-unaligned-load for decent performance, or to use + * Clang instead. * - * This is available on ARMv7-A, but is less efficient than a single - * VZIP.32. + * Fortunately, we can control the first one with a pragma that forces GCC + * into -O2, but the other one we can't control without "failed to inline + * always inline function due to target mismatch" warnings. */ + #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && \ + !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ + #pragma GCC push_options + #pragma GCC optimize("-O2") + #endif - /* - * Function-like macro: - * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t - * &outHi) - * { - - * outLo = (uint32x2_t)(in & 0xFFFFFFFF); - * outHi = (uint32x2_t)(in >> 32); - * in = UNDEFINED; - * } - */ - #if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ - && defined(__GNUC__) && !defined(__aarch64__) && !defined(__arm64__) - #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - \ - /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, \ - * %f0 = upper D half */ \ - /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 \ - */ \ - /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 \ - */ \ - __asm__("vzip.32 %e0, %f0" : "+w"(in)); \ - (outLo) = vget_low_u32(vreinterpretq_u32_u64(in)); \ - (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ - \ - } while (0) + #if XXH_VECTOR == XXH_NEON + /* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an + * upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split + * into two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] | + * '---------. .--------' | | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 + * ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 + * is completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so + * modifying D11 will modify the high half of Q5. This is similar to how + * modifying AH will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd + * lanes in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead + * of taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 + * on the lower and upper halves of the Q register to end up with the + * high and low halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { - #else - #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - \ - (outLo) = vmovn_u64(in); \ - (outHi) = vshrn_n_u64((in), 32); \ - \ - } while (0) + * d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions + * modifying two registers at once is not possible in GCC or Clang's IR, + * and they have to create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, + * many quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes + * to a D-form register zero the high bits, similar to how writes to + * W-form scalar registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the + * equivalent of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to + * only modify one operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be + * this mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], + * v0[1] } zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } zip2 + * v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 + * (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single + * VZIP.32. + */ - #endif - #endif /* XXH_VECTOR == XXH_NEON */ + /*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t + * &outHi) + * { - /* - * VSX and Z Vector helpers. - * - * This is very messy, and any pull requests to clean this up are welcome. - * - * There are a lot of problems with supporting VSX and s390x, due to - * inconsistent intrinsics, spotty coverage, and multiple endiannesses. - */ - #if XXH_VECTOR == XXH_VSX - #if defined(__s390x__) - #include - #else - /* gcc's altivec.h can have the unwanted consequence to unconditionally - * #define bool, vector, and pixel keywords, - * with bad consequences for programs already using these keywords for - * other purposes. The paragraph defining these macros is skipped when - * __APPLE_ALTIVEC__ is defined. - * __APPLE_ALTIVEC__ is _generally_ defined automatically by the - * compiler, but it seems that, in some cases, it isn't. Force the build - * macro to be defined, so that keywords are not altered. + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } */ - #if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) - #define __APPLE_ALTIVEC__ + #if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && defined(__GNUC__) && !defined(__aarch64__) && \ + !defined(__arm64__) + #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, \ + * %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 \ + */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 \ + */ \ + __asm__("vzip.32 %e0, %f0" : "+w"(in)); \ + (outLo) = vget_low_u32(vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + \ + } while (0) + #else + #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + \ + (outLo) = vmovn_u64(in); \ + (outHi) = vshrn_n_u64((in), 32); \ + \ + } while (0) + #endif + #endif /* XXH_VECTOR == XXH_NEON */ + + /* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ + #if XXH_VECTOR == XXH_VSX + #if defined(__s390x__) + #include + #else + /* gcc's altivec.h can have the unwanted consequence to + * unconditionally #define bool, vector, and pixel keywords, with bad + * consequences for programs already using these keywords for other + * purposes. The paragraph defining these macros is skipped when + * __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the + * compiler, but it seems that, in some cases, it isn't. Force the + * build macro to be defined, so that keywords are not altered. + */ + #if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) + #define __APPLE_ALTIVEC__ + #endif + #include #endif - #include - #endif typedef __vector unsigned long long xxh_u64x2; typedef __vector unsigned char xxh_u8x16; typedef __vector unsigned xxh_u32x4; - #ifndef XXH_VSX_BE - #if defined(__BIG_ENDIAN__) || \ - (defined(__BYTE_ORDER__) && \ - __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) - #define XXH_VSX_BE 1 - #elif defined(__VEC_ELEMENT_REG_ORDER__) && \ - __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ - #warning \ - "-maltivec=be is not recommended. Please use native endianness." - #define XXH_VSX_BE 1 - #else - #define XXH_VSX_BE 0 - #endif - #endif /* !defined(XXH_VSX_BE) */ - - #if XXH_VSX_BE - /* A wrapper for POWER9's vec_revb. */ - #if defined(__POWER9_VECTOR__) || \ - (defined(__clang__) && defined(__s390x__)) - #define XXH_vec_revb vec_revb - #else + #ifndef XXH_VSX_BE + #if defined(__BIG_ENDIAN__) || \ + (defined(__BYTE_ORDER__) && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define XXH_VSX_BE 1 + #elif defined(__VEC_ELEMENT_REG_ORDER__) && \ + __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ + #warning \ + "-maltivec=be is not recommended. Please use native endianness." + #define XXH_VSX_BE 1 + #else + #define XXH_VSX_BE 0 + #endif + #endif /* !defined(XXH_VSX_BE) */ + + #if XXH_VSX_BE + #if defined(__POWER9_VECTOR__) || \ + (defined(__clang__) && defined(__s390x__)) + #define XXH_vec_revb vec_revb + #else +/*! + * A polyfill for POWER9's vec_revb(). + */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) { xxh_u8x16 const vByteSwap = {0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, @@ -2834,40 +3458,40 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) { } - #endif - #endif /* XXH_VSX_BE */ + #endif + #endif /* XXH_VSX_BE */ -/* - * Performs an unaligned load and byte swaps it on big endian. +/*! + * Performs an unaligned vector load and byte swaps it on big endian. */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) { xxh_u64x2 ret; memcpy(&ret, ptr, sizeof(xxh_u64x2)); - #if XXH_VSX_BE + #if XXH_VSX_BE ret = XXH_vec_revb(ret); - #endif + #endif return ret; } - /* - * vec_mulo and vec_mule are very problematic intrinsics on PowerPC - * - * These intrinsics weren't added until GCC 8, despite existing for a - * while, and they are endian dependent. Also, their meaning swap - * depending on version. - * */ - #if defined(__s390x__) - /* s390x is always big endian, no issue on this platform */ - #define XXH_vec_mulo vec_mulo - #define XXH_vec_mule vec_mule - #elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) - /* Clang has a better way to control this, we can just use the builtin - * which doesn't swap. */ - #define XXH_vec_mulo __builtin_altivec_vmulouw - #define XXH_vec_mule __builtin_altivec_vmuleuw - #else + /* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a + * while, and they are endian dependent. Also, their meaning swap + * depending on version. + * */ + #if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ + #define XXH_vec_mulo vec_mulo + #define XXH_vec_mule vec_mule + #elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) + /* Clang has a better way to control this, we can just use the builtin + * which doesn't swap. */ + #define XXH_vec_mulo __builtin_altivec_vmulouw + #define XXH_vec_mule __builtin_altivec_vmuleuw + #else /* gcc needs inline assembly */ /* Adapted from * https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ @@ -2887,40 +3511,41 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) { } - #endif /* XXH_vec_mulo, XXH_vec_mule */ - #endif /* XXH_VECTOR == XXH_VSX */ + #endif /* XXH_vec_mulo, XXH_vec_mule */ + #endif /* XXH_VECTOR == XXH_VSX */ - /* prefetch - * can be disabled, by declaring XXH_NO_PREFETCH build macro */ - #if defined(XXH_NO_PREFETCH) - #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ - #else - #if defined(_MSC_VER) && \ - (defined(_M_X64) || \ - defined( \ - _M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ - #include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ - #define XXH_PREFETCH(ptr) _mm_prefetch((const char *)(ptr), _MM_HINT_T0) - #elif defined(__GNUC__) && \ - ((__GNUC__ >= 4) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))) - #define XXH_PREFETCH(ptr) \ - __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) - #else + /* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ + #if defined(XXH_NO_PREFETCH) #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ - #endif - #endif /* XXH_NO_PREFETCH */ + #else + #if defined(_MSC_VER) && \ + (defined(_M_X64) || \ + defined( \ + _M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ + #include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ + #define XXH_PREFETCH(ptr) \ + _mm_prefetch((const char *)(ptr), _MM_HINT_T0) + #elif defined(__GNUC__) && \ + ((__GNUC__ >= 4) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))) + #define XXH_PREFETCH(ptr) \ + __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) + #else + #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ + #endif + #endif /* XXH_NO_PREFETCH */ - /* ========================================== - * XXH3 default settings - * ========================================== */ + /* ========================================== + * XXH3 default settings + * ========================================== */ - #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ - #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) - #error "default keyset is not large enough" - #endif + #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) + #error "default keyset is not large enough" + #endif -/* Pseudorandom secret taken directly from FARSH */ +/*! Pseudorandom secret taken directly from FARSH. */ XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { @@ -2943,69 +3568,79 @@ static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { }; - #ifdef XXH_OLD_NAMES - #define kSecret XXH3_kSecret - #endif + #ifdef XXH_OLD_NAMES + #define kSecret XXH3_kSecret + #endif - /* - * Calculates a 32-bit to 64-bit long multiply. - * - * Wraps __emulu on MSVC x86 because it tends to call __allmul when it - * doesn't need to (but it shouldn't need to anyways, it is about 7 - * instructions to do a 64x64 multiply...). Since we know that this will - * _always_ emit MULL, we use that instead of the normal method. - * - * If you are compiling for platforms like Thumb-1 and don't have a better - * option, you may also want to write your own long multiply routine here. - * - * XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) - * { + #ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it + * doesn't need to (but it shouldn't need to anyways, it is about 7 instructions + * to do a 64x64 multiply...). Since we know that this will _always_ emit + * `MULL`, we use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better + * option, you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { - * return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); - * } - */ - #if defined(_MSC_VER) && defined(_M_IX86) - #include - #define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) - #else - /* - * Downcast + upcast is usually better than masking on older compilers - * like GCC 4.2 (especially 32-bit ones), all without affecting newer - * compilers. - * - * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both - * operands and perform a full 64x64 multiply -- entirely redundant on - * 32-bit. - */ - #define XXH_mult32to64(x, y) \ - ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) - #endif + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); -/* - * Calculates a 64->128-bit long multiply. +} + + #elif defined(_MSC_VER) && defined(_M_IX86) + #include + #define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) + #else + /* + * Downcast + upcast is usually better than masking on older compilers + * like GCC 4.2 (especially 32-bit ones), all without affecting newer + * compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both + * operands and perform a full 64x64 multiply -- entirely redundant on + * 32-bit. + */ + #define XXH_mult32to64(x, y) \ + ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) + #endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. * - * Uses __uint128_t and _umul128 if available, otherwise uses a scalar version. + * @param lhs, rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. */ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { - /* - * GCC/Clang __uint128_t method. - * - * On most 64-bit targets, GCC and Clang define a __uint128_t type. - * This is usually the best way as it usually uses a native long 64-bit - * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. - * - * Usually. - * - * Despite being a 32-bit platform, Clang (and emscripten) define this type - * despite not having the arithmetic for it. This results in a laggy - * compiler builtin call which calculates a full 128-bit multiply. - * In that case it is best to use the portable one. - * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 - */ - #if defined(__GNUC__) && !defined(__wasm__) && \ - defined(__SIZEOF_INT128__) || \ - (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this + * type despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ + #if defined(__GNUC__) && !defined(__wasm__) && \ + defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; XXH128_hash_t r128; @@ -3013,19 +3648,19 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { r128.high64 = (xxh_u64)(product >> 64); return r128; - /* - * MSVC for x64's _umul128 method. - * - * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 - * *HighProduct); - * - * This compiles to single operand MUL on x64. - */ - #elif defined(_M_X64) || defined(_M_IA64) + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 + * *HighProduct); + * + * This compiles to single operand MUL on x64. + */ + #elif defined(_M_X64) || defined(_M_IA64) - #ifndef _MSC_VER - #pragma intrinsic(_umul128) - #endif + #ifndef _MSC_VER + #pragma intrinsic(_umul128) + #endif xxh_u64 product_high; xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); XXH128_hash_t r128; @@ -3033,7 +3668,7 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { r128.high64 = product_high; return r128; - #else + #else /* * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. * @@ -3093,16 +3728,20 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { r128.low64 = lower; r128.high64 = upper; return r128; - #endif + #endif } -/* - * Does a 64-bit to 128-bit multiply, then XOR folds it. +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. * * The reason for the separate function is to prevent passing too many structs * around by value. This will hopefully inline the multiply, but we don't force * it. + * + * @param lhs, rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() */ static xxh_u64 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { @@ -3111,7 +3750,7 @@ static xxh_u64 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { } -/* Seems to produce slightly better code on GCC for some reason. */ +/*! Seems to produce slightly better code on GCC for some reason. */ XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) { XXH_ASSERT(0 <= shift && shift < 64); @@ -3216,7 +3855,7 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_4to8_64b(const xxh_u8 *input, size_t len, XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); - XXH_ASSERT(4 <= len && len < 8); + XXH_ASSERT(4 <= len && len <= 8); seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; { @@ -3239,7 +3878,7 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_9to16_64b(const xxh_u8 *input, XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); - XXH_ASSERT(8 <= len && len <= 16); + XXH_ASSERT(9 <= len && len <= 16); { xxh_u64 const bitflip1 = @@ -3306,11 +3945,10 @@ XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8 *XXH_RESTRICT input, const xxh_u8 *XXH_RESTRICT secret, xxh_u64 seed64) { - #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ - && \ - !defined( \ - XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like \ + XXH32 hack */ /* * UGLY HACK: * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in @@ -3326,8 +3964,8 @@ XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8 *XXH_RESTRICT input, * GCC generates much better scalar code than Clang for the rest of XXH3, * which is why finding a more optimal codepath is an interest. */ - __asm__("" : "+r"(seed64)); - #endif + XXH_COMPILER_GUARD(seed64); + #endif { xxh_u64 const input_lo = XXH_readLE64(input); @@ -3381,7 +4019,7 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_17to128_64b( } - #define XXH3_MIDSIZE_MAX 240 + #define XXH3_MIDSIZE_MAX 240 XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( const xxh_u8 *XXH_RESTRICT input, size_t len, @@ -3391,8 +4029,8 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( (void)secretSize; XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); - #define XXH3_MIDSIZE_STARTOFFSET 3 - #define XXH3_MIDSIZE_LASTOFFSET 17 + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 { @@ -3407,31 +4045,31 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( acc = XXH3_avalanche(acc); XXH_ASSERT(nbRounds >= 8); - #if defined(__clang__) /* Clang */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ - /* - * UGLY HACK: - * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. - * In everywhere else, it uses scalar code. - * - * For 64->128-bit multiplies, even if the NEON was 100% optimal, it - * would still be slower than UMAAL (see XXH_mult64to128). - * - * Unfortunately, Clang doesn't handle the long multiplies properly and - * converts them to the nonexistent "vmulq_u64" intrinsic, which is then - * scalarized into an ugly mess of VMOV.32 instructions. - * - * This mess is difficult to avoid without turning autovectorization - * off completely, but they are usually relatively minor and/or not - * worth it to fix. - * - * This loop is the easiest to fix, as unlike XXH32, this pragma - * _actually works_ because it is a loop vectorization instead of an - * SLP vectorization. - */ - #pragma clang loop vectorize(disable) - #endif + #if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) + #endif for (i = 8; i < nbRounds; i++) { acc += @@ -3450,17 +4088,17 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( } - /* ======= Long Keys ======= */ + /* ======= Long Keys ======= */ - #define XXH_STRIPE_LEN 64 - #define XXH_SECRET_CONSUME_RATE \ - 8 /* nb of secret bytes consumed at each accumulation */ - #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + #define XXH_STRIPE_LEN 64 + #define XXH_SECRET_CONSUME_RATE \ + 8 /* nb of secret bytes consumed at each accumulation */ + #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) - #ifdef XXH_OLD_NAMES - #define STRIPE_LEN XXH_STRIPE_LEN - #define ACC_NB XXH_ACC_NB - #endif + #ifdef XXH_OLD_NAMES + #define STRIPE_LEN XXH_STRIPE_LEN + #define ACC_NB XXH_ACC_NB + #endif XXH_FORCE_INLINE void XXH_writeLE64(void *dst, xxh_u64 v64) { @@ -3469,56 +4107,58 @@ XXH_FORCE_INLINE void XXH_writeLE64(void *dst, xxh_u64 v64) { } - /* Several intrinsic functions below are supposed to accept __int64 as - * argument, as documented in - * https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . However, - * several environments do not define __int64 type, requiring a workaround. - */ - #if !defined(__VMS) && \ - (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ - (__STDC_VERSION__ >= 199901L) /* C99 */)) + /* Several intrinsic functions below are supposed to accept __int64 as + * argument, as documented in + * https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ + #if !defined(__VMS) && \ + (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 199901L) /* C99 */)) typedef int64_t xxh_i64; - #else + #else /* the following type must have a width of 64-bit */ typedef long long xxh_i64; - #endif + #endif - /* - * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the - * most optimized. - * - * It is a hardened version of UMAC, based off of FARSH's implementation. - * - * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD - * implementations, and it is ridiculously fast. - * - * We harden it by mixing the original input to the accumulators as well as - * the product. - * - * This means that in the (relatively likely) case of a multiply by zero, the - * original input is preserved. - * - * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve - * cross-pollination, as otherwise the upper and lower halves would be - * essentially independent. - * - * This doesn't matter on 64-bit hashes since they all get merged together in - * the end, so we skip the extra step. - * - * Both XXH3_64bits and XXH3_128bits use this subroutine. - */ + /* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the + * most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as + * the product. + * + * This means that in the (relatively likely) case of a multiply by zero, + * the original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together + * in the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ - #if (XXH_VECTOR == XXH_AVX512) || defined(XXH_X86DISPATCH) + #if (XXH_VECTOR == XXH_AVX512) || \ + (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) - #ifndef XXH_TARGET_AVX512 - #define XXH_TARGET_AVX512 /* disable attribute target */ - #endif + #ifndef XXH_TARGET_AVX512 + #define XXH_TARGET_AVX512 /* disable attribute target */ + #endif XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_accumulate_512_avx512( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, const void *XXH_RESTRICT secret) { - XXH_ALIGN(64) __m512i *const xacc = (__m512i *)acc; + __m512i *const xacc = (__m512i *)acc; XXH_ASSERT((((size_t)acc) & 63) == 0); XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); @@ -3576,8 +4216,8 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_scrambleAcc_avx512( XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); { - XXH_ALIGN(64) __m512i *const xacc = (__m512i *)acc; - const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + __m512i *const xacc = (__m512i *)acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); /* xacc[0] ^= (xacc[0] >> 47) */ __m512i const acc_vec = *xacc; @@ -3609,19 +4249,21 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512( int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); __m512i const seed = _mm512_mask_set1_epi64( - _mm512_set1_epi64((xxh_i64)seed64), 0xAA, -(xxh_i64)seed64); + _mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); - XXH_ALIGN(64) const __m512i *const src = (const __m512i *)XXH3_kSecret; - XXH_ALIGN(64) __m512i *const dest = (__m512i *)customSecret; - int i; + const __m512i *const src = (const __m512i *)((const void *)XXH3_kSecret); + __m512i *const dest = (__m512i *)customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); for (i = 0; i < nbRounds; ++i) { /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void - * const*', this will warn "discards ‘const’ qualifier". */ + * const*', this will warn "discards 'const' qualifier". */ union { - XXH_ALIGN(64) const __m512i *cp; - XXH_ALIGN(64) void *p; + const __m512i *cp; + void * p; } remote_const_void; @@ -3635,13 +4277,14 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512( } - #endif + #endif - #if (XXH_VECTOR == XXH_AVX2) || defined(XXH_X86DISPATCH) + #if (XXH_VECTOR == XXH_AVX2) || \ + (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) - #ifndef XXH_TARGET_AVX2 - #define XXH_TARGET_AVX2 /* disable attribute target */ - #endif + #ifndef XXH_TARGET_AVX2 + #define XXH_TARGET_AVX2 /* disable attribute target */ + #endif XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, @@ -3650,7 +4293,7 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2( XXH_ASSERT((((size_t)acc) & 31) == 0); { - XXH_ALIGN(32) __m256i *const xacc = (__m256i *)acc; + __m256i *const xacc = (__m256i *)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ @@ -3692,7 +4335,7 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_scrambleAcc_avx2( XXH_ASSERT((((size_t)acc) & 31) == 0); { - XXH_ALIGN(32) __m256i *const xacc = (__m256i *)acc; + __m256i *const xacc = (__m256i *)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ const __m256i *const xsecret = (const __m256i *)secret; @@ -3732,24 +4375,23 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2( XXH_PREFETCH(customSecret); { - __m256i const seed = _mm256_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64, - -(xxh_i64)seed64, (xxh_i64)seed64); + __m256i const seed = + _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, + (xxh_i64)(0U - seed64), (xxh_i64)seed64); - XXH_ALIGN(64) const __m256i *const src = (const __m256i *)XXH3_kSecret; - XXH_ALIGN(64) __m256i * dest = (__m256i *)customSecret; + const __m256i *const src = (const __m256i *)((const void *)XXH3_kSecret); + __m256i * dest = (__m256i *)customSecret; - #if defined(__GNUC__) || defined(__clang__) + #if defined(__GNUC__) || defined(__clang__) /* * On GCC & Clang, marking 'dest' as modified will cause the compiler: * - do not extract the secret from sse registers in the internal loop * - use less common registers, and avoid pushing these reg into stack - * The asm hack causes Clang to assume that XXH3_kSecretPtr aliases with - * customSecret, and on aarch64, this prevented LDP from merging two - * loads together for free. Putting the loads together before the stores - * properly generates LDP. */ - __asm__("" : "+r"(dest)); - #endif + XXH_COMPILER_GUARD(dest); + #endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); /* GCC -O2 need unroll loop manually */ dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src + 0), seed); @@ -3763,13 +4405,14 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2( } - #endif + #endif - #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + /* x86dispatch always generates SSE2 */ + #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) - #ifndef XXH_TARGET_SSE2 - #define XXH_TARGET_SSE2 /* disable attribute target */ - #endif + #ifndef XXH_TARGET_SSE2 + #define XXH_TARGET_SSE2 /* disable attribute target */ + #endif XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_accumulate_512_sse2( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, @@ -3779,7 +4422,7 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_accumulate_512_sse2( XXH_ASSERT((((size_t)acc) & 15) == 0); { - XXH_ALIGN(16) __m128i *const xacc = (__m128i *)acc; + __m128i *const xacc = (__m128i *)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i *const xinput = (const __m128i *)input; @@ -3820,7 +4463,7 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_scrambleAcc_sse2( XXH_ASSERT((((size_t)acc) & 15) == 0); { - XXH_ALIGN(16) __m128i *const xacc = (__m128i *)acc; + __m128i *const xacc = (__m128i *)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i *const xsecret = (const __m128i *)secret; @@ -3859,30 +4502,34 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2( int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); - #if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 - // MSVC 32bit mode does not support _mm_set_epi64x before 2015 + #if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ XXH_ALIGN(16) - const xxh_i64 seed64x2[2] = {(xxh_i64)seed64, -(xxh_i64)seed64}; + const xxh_i64 seed64x2[2] = {(xxh_i64)seed64, (xxh_i64)(0U - seed64)}; __m128i const seed = _mm_load_si128((__m128i const *)seed64x2); - #else - __m128i const seed = _mm_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64); - #endif + #else + __m128i const seed = + _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); + #endif int i; - XXH_ALIGN(64) const float *const src = (float const *)XXH3_kSecret; - XXH_ALIGN(XXH_SEC_ALIGN) __m128i *dest = (__m128i *)customSecret; - #if defined(__GNUC__) || defined(__clang__) + const void *const src16 = XXH3_kSecret; + __m128i * dst16 = (__m128i *)customSecret; + #if defined(__GNUC__) || defined(__clang__) /* * On GCC & Clang, marking 'dest' as modified will cause the compiler: * - do not extract the secret from sse registers in the internal loop * - use less common registers, and avoid pushing these reg into stack */ - __asm__("" : "+r"(dest)); - #endif + XXH_COMPILER_GUARD(dst16); + #endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); for (i = 0; i < nbRounds; ++i) { - dest[i] = _mm_add_epi64(_mm_castps_si128(_mm_load_ps(src + i * 4)), seed); + dst16[i] = + _mm_add_epi64(_mm_load_si128((const __m128i *)src16 + i), seed); } @@ -3890,9 +4537,9 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2( } - #endif + #endif - #if (XXH_VECTOR == XXH_NEON) + #if (XXH_VECTOR == XXH_NEON) XXH_FORCE_INLINE void XXH3_accumulate_512_neon( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, @@ -3901,7 +4548,7 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_neon( XXH_ASSERT((((size_t)acc) & 15) == 0); { - XXH_ALIGN(16) uint64x2_t *const xacc = (uint64x2_t *)acc; + uint64x2_t *const xacc = (uint64x2_t *)acc; /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ uint8_t const *const xinput = (const uint8_t *)input; @@ -3996,9 +4643,9 @@ XXH_FORCE_INLINE void XXH3_scrambleAcc_neon(void *XXH_RESTRICT acc, } - #endif + #endif - #if (XXH_VECTOR == XXH_VSX) + #if (XXH_VECTOR == XXH_VSX) XXH_FORCE_INLINE void XXH3_accumulate_512_vsx(void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, @@ -4025,12 +4672,12 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_vsx(void *XXH_RESTRICT acc, xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); xacc[i] += product; - /* swap high and low halves */ - #ifdef __s390x__ + /* swap high and low halves */ + #ifdef __s390x__ xacc[i] += vec_permi(data_vec, data_vec, 2); - #else + #else xacc[i] += vec_xxpermdi(data_vec, data_vec, 2); - #endif + #endif } @@ -4075,7 +4722,7 @@ XXH_FORCE_INLINE void XXH3_scrambleAcc_vsx(void *XXH_RESTRICT acc, } - #endif + #endif /* scalar variants - universal */ @@ -4083,7 +4730,6 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_scalar( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, const void *XXH_RESTRICT secret) { - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 *const xacc = (xxh_u64 *)acc; /* presumed aligned */ const xxh_u8 *const xinput = (const xxh_u8 *)input; /* no alignment restriction */ @@ -4105,7 +4751,6 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_scalar( XXH_FORCE_INLINE void XXH3_scrambleAcc_scalar(void *XXH_RESTRICT acc, const void *XXH_RESTRICT secret) { - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 *const xacc = (xxh_u64 *)acc; /* presumed aligned */ const xxh_u8 *const xsecret = (const xxh_u8 *)secret; /* no alignment restriction */ @@ -4135,7 +4780,7 @@ XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( const xxh_u8 *kSecretPtr = XXH3_kSecret; XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); - #if defined(__clang__) && defined(__aarch64__) + #if defined(__clang__) && defined(__aarch64__) /* * UGLY HACK: * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are @@ -4164,8 +4809,8 @@ XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( * without hack: 2654.4 MB/s * with hack: 3202.9 MB/s */ - __asm__("" : "+r"(kSecretPtr)); - #endif + XXH_COMPILER_GUARD(kSecretPtr); + #endif /* * Note: in debug mode, this overrides the asm optimization * and Clang will emit MOVK chains again. @@ -4200,55 +4845,55 @@ typedef void (*XXH3_f_accumulate_512)(void *XXH_RESTRICT, const void *, typedef void (*XXH3_f_scrambleAcc)(void *XXH_RESTRICT, const void *); typedef void (*XXH3_f_initCustomSecret)(void *XXH_RESTRICT, xxh_u64); - #if (XXH_VECTOR == XXH_AVX512) + #if (XXH_VECTOR == XXH_AVX512) - #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 - #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 - #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 + #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 + #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 - #elif (XXH_VECTOR == XXH_AVX2) + #elif (XXH_VECTOR == XXH_AVX2) - #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 - #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 - #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 + #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 + #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 - #elif (XXH_VECTOR == XXH_SSE2) + #elif (XXH_VECTOR == XXH_SSE2) - #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 - #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 - #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 + #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 + #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 - #elif (XXH_VECTOR == XXH_NEON) + #elif (XXH_VECTOR == XXH_NEON) - #define XXH3_accumulate_512 XXH3_accumulate_512_neon - #define XXH3_scrambleAcc XXH3_scrambleAcc_neon - #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #define XXH3_accumulate_512 XXH3_accumulate_512_neon + #define XXH3_scrambleAcc XXH3_scrambleAcc_neon + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - #elif (XXH_VECTOR == XXH_VSX) + #elif (XXH_VECTOR == XXH_VSX) - #define XXH3_accumulate_512 XXH3_accumulate_512_vsx - #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx - #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #define XXH3_accumulate_512 XXH3_accumulate_512_vsx + #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - #else /* scalar */ + #else /* scalar */ - #define XXH3_accumulate_512 XXH3_accumulate_512_scalar - #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar - #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #define XXH3_accumulate_512 XXH3_accumulate_512_scalar + #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - #endif + #endif - #ifndef XXH_PREFETCH_DIST - #ifdef __clang__ - #define XXH_PREFETCH_DIST 320 - #else - #if (XXH_VECTOR == XXH_AVX512) - #define XXH_PREFETCH_DIST 512 + #ifndef XXH_PREFETCH_DIST + #ifdef __clang__ + #define XXH_PREFETCH_DIST 320 #else - #define XXH_PREFETCH_DIST 384 - #endif - #endif /* __clang__ */ - #endif /* XXH_PREFETCH_DIST */ + #if (XXH_VECTOR == XXH_AVX512) + #define XXH_PREFETCH_DIST 512 + #else + #define XXH_PREFETCH_DIST 384 + #endif + #endif /* __clang__ */ + #endif /* XXH_PREFETCH_DIST */ /* * XXH3_accumulate() @@ -4308,8 +4953,9 @@ XXH_FORCE_INLINE void XXH3_hashLong_internal_loop( { const xxh_u8 *const p = input + len - XXH_STRIPE_LEN; - #define XXH_SECRET_LASTACC_START \ - 7 /* not aligned on 8, last secret is different from acc & scrambler */ + #define XXH_SECRET_LASTACC_START \ + 7 /* not aligned on 8, last secret is different from acc & scrambler \ + */ f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); @@ -4337,10 +4983,10 @@ static XXH64_hash_t XXH3_mergeAccs(const xxh_u64 *XXH_RESTRICT acc, for (i = 0; i < 4; i++) { result64 += XXH3_mix2Accs(acc + 2 * i, secret + 16 * i); - #if defined(__clang__) /* Clang */ \ - && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + #if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ /* * UGLY HACK: * Prevent autovectorization on Clang ARMv7-a. Exact same problem as @@ -4349,8 +4995,8 @@ static XXH64_hash_t XXH3_mergeAccs(const xxh_u64 *XXH_RESTRICT acc, * without hack: 2063.7 MB/s * with hack: 2560.7 MB/s */ - __asm__("" : "+r"(result64)); - #endif + XXH_COMPILER_GUARD(result64); + #endif } @@ -4358,13 +5004,13 @@ static XXH64_hash_t XXH3_mergeAccs(const xxh_u64 *XXH_RESTRICT acc, } - #define XXH3_INIT_ACC \ - { \ - \ - XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ - XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 \ - \ - } + #define XXH3_INIT_ACC \ + { \ + \ + XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 \ + \ + } XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal( const void *XXH_RESTRICT input, size_t len, const void *XXH_RESTRICT secret, @@ -4379,9 +5025,9 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal( /* converge into final hash */ XXH_STATIC_ASSERT(sizeof(acc) == 64); - /* do not align on 8, so that the secret is different from the accumulator - */ - #define XXH_SECRET_MERGEACCS_START 11 + /* do not align on 8, so that the secret is different from the accumulator + */ + #define XXH_SECRET_MERGEACCS_START 11 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); return XXH3_mergeAccs(acc, (const xxh_u8 *)secret + XXH_SECRET_MERGEACCS_START, @@ -4501,6 +5147,7 @@ XXH3_64bits_internal(const void *XXH_RESTRICT input, size_t len, /* === Public entry point === */ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *input, size_t len) { return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), @@ -4508,6 +5155,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *input, size_t len) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *input, size_t len, const void *secret, @@ -4518,6 +5166,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *input, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void *input, size_t len, XXH64_hash_t seed) { @@ -4603,6 +5252,7 @@ static void XXH_alignedFree(void *p) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void) { XXH3_state_t *const state = @@ -4613,6 +5263,7 @@ XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr) { XXH_alignedFree(statePtr); @@ -4620,6 +5271,7 @@ XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t * dst_state, const XXH3_state_t *src_state) { @@ -4627,9 +5279,8 @@ XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t * dst_state, } -static void XXH3_64bits_reset_internal(XXH3_state_t *statePtr, - XXH64_hash_t seed, const void *secret, - size_t secretSize) { +static void XXH3_reset_internal(XXH3_state_t *statePtr, XXH64_hash_t seed, + const void *secret, size_t secretSize) { size_t const initStart = offsetof(XXH3_state_t, bufferedSize); size_t const initLength = @@ -4654,26 +5305,28 @@ static void XXH3_64bits_reset_internal(XXH3_state_t *statePtr, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t *statePtr) { if (statePtr == NULL) return XXH_ERROR; - XXH3_64bits_reset_internal(statePtr, 0, XXH3_kSecret, - XXH_SECRET_DEFAULT_SIZE); + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret( XXH3_state_t *statePtr, const void *secret, size_t secretSize) { if (statePtr == NULL) return XXH_ERROR; - XXH3_64bits_reset_internal(statePtr, 0, secret, secretSize); + XXH3_reset_internal(statePtr, 0, secret, secretSize); if (secret == NULL) return XXH_ERROR; if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t *statePtr, XXH64_hash_t seed) { @@ -4681,7 +5334,7 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t *statePtr, if (seed == 0) return XXH3_64bits_reset(statePtr); if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); - XXH3_64bits_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } @@ -4733,12 +5386,12 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, XXH3_f_scrambleAcc f_scramble) { if (input == NULL) - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) + #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ + (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) return XXH_OK; - #else + #else return XXH_ERROR; - #endif + #endif { @@ -4747,6 +5400,7 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, (state->extSecret == NULL) ? state->customSecret : state->extSecret; state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { /* fill in tmp buffer */ @@ -4756,10 +5410,10 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, } - /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ - #define XXH3_INTERNALBUFFER_STRIPES \ - (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + #define XXH3_INTERNALBUFFER_STRIPES \ + (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ @@ -4783,7 +5437,7 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, XXH_ASSERT(input < bEnd); /* Consume input by a multiple of internal buffer size */ - if (input + XXH3_INTERNALBUFFER_SIZE < bEnd) { + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { const xxh_u8 *const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; do { @@ -4814,6 +5468,7 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update(XXH3_state_t *state, const void *input, size_t len) { @@ -4859,6 +5514,7 @@ XXH_FORCE_INLINE void XXH3_digest_long(XXH64_hash_t * acc, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *state) { const unsigned char *const secret = @@ -4881,8 +5537,9 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *state) { } - #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) +/*! @ingroup xxh3_family */ XXH_PUBLIC_API void XXH3_generateSecret(void * secretBuffer, const void *customSeed, size_t customSeedSize) { @@ -5398,6 +6055,7 @@ XXH3_128bits_internal(const void *input, size_t len, XXH64_hash_t seed64, /* === Public XXH128 API === */ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void *input, size_t len) { return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, @@ -5406,6 +6064,7 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void *input, size_t len) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *input, size_t len, const void *secret, @@ -5416,6 +6075,7 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *input, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void * input, size_t len, XXH64_hash_t seed) { @@ -5426,6 +6086,7 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void * input, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *input, size_t len, XXH64_hash_t seed) { @@ -5437,37 +6098,31 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *input, size_t len, /* * All the functions are actually the same as for 64-bit streaming variant. - * The only difference is the finalizatiom routine. + * The only difference is the finalization routine. */ -static void XXH3_128bits_reset_internal(XXH3_state_t *statePtr, - XXH64_hash_t seed, const void *secret, - size_t secretSize) { - - XXH3_64bits_reset_internal(statePtr, seed, secret, secretSize); - -} - +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t *statePtr) { if (statePtr == NULL) return XXH_ERROR; - XXH3_128bits_reset_internal(statePtr, 0, XXH3_kSecret, - XXH_SECRET_DEFAULT_SIZE); + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret( XXH3_state_t *statePtr, const void *secret, size_t secretSize) { if (statePtr == NULL) return XXH_ERROR; - XXH3_128bits_reset_internal(statePtr, 0, secret, secretSize); + XXH3_reset_internal(statePtr, 0, secret, secretSize); if (secret == NULL) return XXH_ERROR; if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t *statePtr, XXH64_hash_t seed) { @@ -5475,11 +6130,12 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t *statePtr, if (seed == 0) return XXH3_128bits_reset(statePtr); if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); - XXH3_128bits_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t *state, const void * input, size_t len) { @@ -5489,6 +6145,7 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t *state, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t *state) { const unsigned char *const secret = @@ -5524,11 +6181,12 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t *state) { } - /* 128-bit utility functions */ + /* 128-bit utility functions */ - #include /* memcmp, memcpy */ + #include /* memcmp, memcpy */ /* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { /* note : XXH128_hash_t is compact, it has no padding byte */ @@ -5540,6 +6198,7 @@ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { * return : >0 if *h128_1 > *h128_2 * <0 if *h128_1 < *h128_2 * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API int XXH128_cmp(const void *h128_1, const void *h128_2) { XXH128_hash_t const h1 = *(const XXH128_hash_t *)h128_1; @@ -5552,6 +6211,7 @@ XXH_PUBLIC_API int XXH128_cmp(const void *h128_1, const void *h128_2) { } /*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t *dst, XXH128_hash_t hash) { @@ -5568,6 +6228,7 @@ XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t *dst, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t *src) { @@ -5578,16 +6239,21 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src) { } - /* Pop our optimization override from above */ - #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ - && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__OPTIMIZE__) && \ - !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ - #pragma GCC pop_options - #endif + /* Pop our optimization override from above */ + #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && \ + !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ + #pragma GCC pop_options + #endif - #endif /* XXH_NO_LONG_LONG */ + #endif /* XXH_NO_LONG_LONG */ + + #endif /* XXH_NO_XXH3 */ +/*! + * @} + */ #endif /* XXH_IMPLEMENTATION */ #if defined(__cplusplus) diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 65bc16bc..06268d18 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -158,7 +158,7 @@ static void at_exit(int signal) { } -#define default_hash(a, b) XXH64(a, b, HASH_CONST) +#define default_hash(a, b) XXH3_64bits(a, b) /* Uninspired gcc plugin instrumentation */ diff --git a/src/afl-performance.c b/src/afl-performance.c index 89b170eb..ff8dfeb6 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -90,7 +90,8 @@ inline u32 hash32(u8 *key, u32 len, u32 seed) { #endif - return (u32)XXH64(key, len, seed); + (void)seed; + return (u32)XXH3_64bits(key, len); } @@ -102,7 +103,8 @@ inline u64 hash64(u8 *key, u32 len, u64 seed) { #endif - return XXH64(key, len, seed); + (void)seed; + return XXH3_64bits(key, len); } -- cgit 1.4.1 From 92c91f25f0abc2eb4b8eccbe9b89f433a08eeaba Mon Sep 17 00:00:00 2001 From: tl455047 Date: Sun, 7 Nov 2021 20:28:03 +0800 Subject: Fix index of cmp_map->log[key]. --- src/afl-fuzz-redqueen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index f6a7f3e6..a1d6e021 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -1669,7 +1669,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, for (j = 0; j < i; ++j) { if (afl->shm.cmp_map->log[key][j].v0 == o->v0 && - afl->shm.cmp_map->log[key][i].v1 == o->v1) { + afl->shm.cmp_map->log[key][j].v1 == o->v1) { goto cmp_fuzz_next_iter; -- cgit 1.4.1 From 93852a23128d24ebb15e3afdbeee07f8664da4ec Mon Sep 17 00:00:00 2001 From: tl455047 Date: Sun, 7 Nov 2021 20:56:40 +0800 Subject: code-format --- src/afl-forkserver.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index b1769bfb..44b6c6f9 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -1017,7 +1017,8 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" - " Tip: you can use https://jwilk.net/software/recidivm to quickly\n" + " Tip: you can use https://jwilk.net/software/recidivm to " + "quickly\n" " estimate the required amount of virtual memory for the " "binary.\n\n" -- cgit 1.4.1 From 75821d2943ee207a6d862440ac1c466b96a52861 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 9 Nov 2021 14:37:14 +0100 Subject: fix afl- tools for new frida --- src/afl-analyze.c | 2 +- src/afl-showmap.c | 2 +- src/afl-tmin.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 60cb1434..d52a6d6e 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -995,7 +995,7 @@ int main(int argc, char **argv_orig, char **envp) { frida_mode = 1; fsrv.frida_mode = frida_mode; - setenv("AFL_FRIDA_INST_SEED", "0x0", 1); + setenv("AFL_FRIDA_INST_SEED", "1", 1); break; diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 23ec0df0..6c06c476 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1056,7 +1056,7 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->frida_mode) { FATAL("Multiple -O options not supported"); } fsrv->frida_mode = true; - setenv("AFL_FRIDA_INST_SEED", "0x0", 1); + setenv("AFL_FRIDA_INST_SEED", "1", 1); break; diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 8ce4bdd5..3f6f14f9 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -1038,7 +1038,7 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->frida_mode) { FATAL("Multiple -O options not supported"); } fsrv->frida_mode = 1; - setenv("AFL_FRIDA_INST_SEED", "0x0", 1); + setenv("AFL_FRIDA_INST_SEED", "1", 1); break; -- cgit 1.4.1 From 9100f3c416707d926fc100d4441cf32bb1da6dd6 Mon Sep 17 00:00:00 2001 From: Akira Moroo Date: Tue, 22 Jun 2021 05:34:20 +0000 Subject: Add initial CoreSight mode support The original code is: https://github.com/RICSecLab/AFLplusplus-cs/tree/retrage/coresight-mode-pr Signed-off-by: Akira Moroo --- .gitignore | 3 + .gitmodules | 6 ++ GNUmakefile | 16 ++- coresight_mode/.gitignore | 2 + coresight_mode/GNUmakefile | 59 +++++++++++ coresight_mode/Makefile | 21 ++++ coresight_mode/README.md | 58 ++++++++++ coresight_mode/coresight-trace | 1 + coresight_mode/patchelf | 1 + .../patches/0001-Add-AFL-forkserver.patch | 117 +++++++++++++++++++++ include/common.h | 1 + include/forkserver.h | 2 + src/afl-analyze.c | 18 +++- src/afl-common.c | 29 +++++ src/afl-fuzz-init.c | 5 +- src/afl-fuzz-stats.c | 9 +- src/afl-fuzz.c | 28 ++++- src/afl-showmap.c | 22 +++- src/afl-tmin.c | 16 ++- 19 files changed, 397 insertions(+), 17 deletions(-) create mode 100644 coresight_mode/.gitignore create mode 100644 coresight_mode/GNUmakefile create mode 100644 coresight_mode/Makefile create mode 100644 coresight_mode/README.md create mode 160000 coresight_mode/coresight-trace create mode 160000 coresight_mode/patchelf create mode 100644 coresight_mode/patches/0001-Add-AFL-forkserver.patch (limited to 'src') diff --git a/.gitignore b/.gitignore index 5268bb37..22ee6bf1 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ afl-g++-fast afl-gotcpu afl-ld afl-ld-lto +afl-cs-proxy afl-qemu-trace afl-showmap afl-tmin @@ -94,3 +95,5 @@ utils/optimin/optimin utils/persistent_mode/persistent_demo utils/persistent_mode/persistent_demo_new utils/persistent_mode/test-instr +!coresight_mode +!coresight_mode/coresight-trace diff --git a/.gitmodules b/.gitmodules index 200f3ecc..cd9d73e9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,9 @@ [submodule "utils/optimin/EvalMaxSAT"] path = utils/optimin/EvalMaxSAT url = https://github.com/FlorentAvellaneda/EvalMaxSAT +[submodule "coresight_mode/patchelf"] + path = coresight_mode/patchelf + url = https://github.com/NixOS/patchelf.git +[submodule "coresight_mode/coresight-trace"] + path = coresight_mode/coresight-trace + url = git@github.com:RICSecLab/coresight-trace.git diff --git a/GNUmakefile b/GNUmakefile index 06840786..ab57e7ad 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -346,7 +346,7 @@ help: @echo "HELP --- the following make targets exist:" @echo "==========================================" @echo "all: just the main afl++ binaries" - @echo "binary-only: everything for binary-only fuzzing: qemu_mode, unicorn_mode, libdislocator, libtokencap" + @echo "binary-only: everything for binary-only fuzzing: coresight_mode, qemu_mode, unicorn_mode, libdislocator, libtokencap" @echo "source-only: everything for source code fuzzing: gcc_plugin, libdislocator, libtokencap" @echo "distrib: everything (for both binary-only and source code fuzzing)" @echo "man: creates simple man pages from the help option of the programs" @@ -564,7 +564,7 @@ all_done: test_build .PHONY: clean clean: - rm -rf $(PROGS) libradamsa.so afl-fuzz-document afl-as as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-qemu-trace afl-gcc-fast afl-gcc-pass.so afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand *.dSYM + rm -rf $(PROGS) libradamsa.so afl-fuzz-document afl-as as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-cs-proxy afl-qemu-trace afl-gcc-fast afl-gcc-pass.so afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand *.dSYM -$(MAKE) -f GNUmakefile.llvm clean -$(MAKE) -f GNUmakefile.gcc_plugin clean $(MAKE) -C utils/libdislocator clean @@ -579,19 +579,23 @@ clean: $(MAKE) -C qemu_mode/libqasan clean -$(MAKE) -C frida_mode clean ifeq "$(IN_REPO)" "1" + -test -e coresight_mode/coresight-trace/Makefile && $(MAKE) -C coresight_mode/coresight-trace clean || true -test -e qemu_mode/qemuafl/Makefile && $(MAKE) -C qemu_mode/qemuafl clean || true test -e unicorn_mode/unicornafl/Makefile && $(MAKE) -C unicorn_mode/unicornafl clean || true else + rm -rf coresight_mode/coresight_trace rm -rf qemu_mode/qemuafl rm -rf unicorn_mode/unicornafl endif .PHONY: deepclean deepclean: clean + rm -rf coresight_mode/coresight-trace rm -rf unicorn_mode/unicornafl rm -rf qemu_mode/qemuafl ifeq "$(IN_REPO)" "1" # NEVER EVER ACTIVATE THAT!!!!! git reset --hard >/dev/null 2>&1 || true + git checkout coresight_mode/coresight-trace git checkout unicorn_mode/unicornafl git checkout qemu_mode/qemuafl endif @@ -610,6 +614,9 @@ endif # -$(MAKE) -C utils/plot_ui -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" +ifeq "$(ARCH)" "aarch64" + -$(MAKE) -C coresight_mode +endif -cd qemu_mode && sh ./build_qemu_support.sh -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif @@ -624,6 +631,9 @@ binary-only: test_shm test_python ready $(PROGS) # -$(MAKE) -C utils/plot_ui -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" +ifeq "$(ARCH)" "aarch64" + -$(MAKE) -C coresight_mode +endif -cd qemu_mode && sh ./build_qemu_support.sh -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif @@ -695,7 +705,7 @@ endif .PHONY: uninstall uninstall: - -cd $${DESTDIR}$(BIN_PATH) && rm -f $(PROGS) $(SH_PROGS) afl-qemu-trace afl-plot-ui afl-fuzz-document afl-network-server afl-g* afl-plot.sh afl-as afl-ld-lto afl-c* afl-lto* + -cd $${DESTDIR}$(BIN_PATH) && rm -f $(PROGS) $(SH_PROGS) afl-cs-proxy afl-qemu-trace afl-plot-ui afl-fuzz-document afl-network-server afl-g* afl-plot.sh afl-as afl-ld-lto afl-c* afl-lto* -cd $${DESTDIR}$(HELPER_PATH) && rm -f afl-g*.*o afl-llvm-*.*o afl-compiler-*.*o libdislocator.so libtokencap.so libcompcov.so libqasan.so afl-frida-trace.so socketfuzz*.so argvfuzz*.so libAFLDriver.a libAFLQemuDriver.a as afl-as SanitizerCoverage*.so compare-transform-pass.so cmplog-*-pass.so split-*-pass.so dynamic_list.txt -rm -rf $${DESTDIR}$(MISC_PATH)/testcases $${DESTDIR}$(MISC_PATH)/dictionaries -sh -c "ls docs/*.md | sed 's|^docs/|$${DESTDIR}$(DOC_PATH)/|' | xargs rm -f" diff --git a/coresight_mode/.gitignore b/coresight_mode/.gitignore new file mode 100644 index 00000000..dedb1613 --- /dev/null +++ b/coresight_mode/.gitignore @@ -0,0 +1,2 @@ +.local +glibc* diff --git a/coresight_mode/GNUmakefile b/coresight_mode/GNUmakefile new file mode 100644 index 00000000..69b72f91 --- /dev/null +++ b/coresight_mode/GNUmakefile @@ -0,0 +1,59 @@ +#!/usr/bin/env make +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2021 Ricerca Security, Inc. All rights reserved. + +SHELL:=bash +PREFIX?=$(shell pwd)/.local + +CS_TRACE:=coresight-trace + +PATCHELF?=$(PREFIX)/bin/patchelf + +GLIBC_VER:=2.33 +GLIBC_NAME:=glibc-$(GLIBC_VER) +GLIBC_URL_BASE:=http://ftp.gnu.org/gnu/glibc +GLIBC_PATCH:=patches/0002-glibc-Add-AFL-forkserver.patch +GLIBC_LDSO?=$(PREFIX)/lib/ld-linux-aarch64.so.1 + +OUTPUT?="$(TARGET).patched" + +all: build + +build: + git submodule update --init --recursive $(CS_TRACE) + $(MAKE) -C $(CS_TRACE) + cp $(CS_TRACE)/cs-proxy ../afl-cs-proxy + +patch: | $(PATCHELF) $(GLIBC_LDSO) + @if test -z "$(TARGET)"; then echo "TARGET is not set"; exit 1; fi + $(PATCHELF) \ + --set-interpreter $(GLIBC_LDSO) \ + --set-rpath $(dir $(GLIBC_LDSO)) \ + --output $(OUTPUT) \ + $(TARGET) + +$(PATCHELF): patchelf + git submodule update --init $< + cd $< && \ + ./bootstrap.sh && \ + ./configure --prefix=$(PREFIX) && \ + $(MAKE) && \ + $(MAKE) check && \ + $(MAKE) install + +$(GLIBC_LDSO): | $(GLIBC_NAME).tar.xz + tar -xf $(GLIBC_NAME).tar.xz + patch -p1 < $(GLIBC_PATCH) + mkdir -p $(GLIBC_NAME)/build + cd $(GLIBC_NAME)/build && \ + ../configure --prefix=$(PREFIX) && \ + $(MAKE) && \ + $(MAKE) install + +$(GLIBC_NAME).tar.xz: + wget -O $@ $(GLIBC_URL_BASE)/$@ + +clean: + $(MAKE) -C $(CS_TRACE) clean + +.PHONY: all build patch clean diff --git a/coresight_mode/Makefile b/coresight_mode/Makefile new file mode 100644 index 00000000..fb8990b9 --- /dev/null +++ b/coresight_mode/Makefile @@ -0,0 +1,21 @@ +#!/usr/bin/env make +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2021 Ricerca Security, Inc. All rights reserved. + +all: + @echo trying to use GNU make... + @gmake all || echo please install GNUmake + +build: + @echo trying to use GNU make... + @gmake build || echo please install GNUmake + +patch: + @echo trying to use GNU make... + @gmake patch || echo please install GNUmake + +clean: + @echo trying to use GNU make... + @gmake clean || echo please install GNUmake + +.PHONY: all build patch clean diff --git a/coresight_mode/README.md b/coresight_mode/README.md new file mode 100644 index 00000000..e08421ed --- /dev/null +++ b/coresight_mode/README.md @@ -0,0 +1,58 @@ +# AFL++ CoreSight mode + +CoreSight mode enables binary-only fuzzing on ARM64 Linux using CoreSight. + +NOTE: CoreSight mode is in the early development stage. Not applicable for production use. + +## Getting started + +Please read the [RICSec/coresight-trace README](https://github.com/RICSecLab/coresight-trace/blob/master/README.md) and check the prerequisites before getting started. + +CoreSight mode supports the AFL fork server mode to reduce `exec` system call overhead. To support it for binary-only fuzzing, it needs to modify the target ELF binary to re-link to the patched glibc. We employ this design from [PTrix](https://github.com/junxzm1990/afl-pt). + +Check out all the git submodules in the `cs_mode` directory: + +```bash +git submodule update --init --recursive +``` + +### Build coresight-trace + +There are some notes on building coresight-trace. Refer to the [README](https://github.com/RICSecLab/coresight-trace/blob/master/README.md) for the details. Run make in the `cs_mode` directory: + +```bash +make build +``` + +Make sure `cs-proxy` is placed in the AFL++ root directory as `afl-cs-proxy`. + +### Patch COTS binary + +The fork server mode requires patchelf and the patched glibc. The dependency build can be done by just run make: + +```bash +make patch TARGET=$BIN +``` + +The above make command builds and installs the dependencies to `$PREFIX` (default to `$PWD/.local`) at the first time. Then, it runs `patchelf` to `$BIN` with output `$OUTPUT` (`$BIN.patched` by default). + +### Run afl-fuzz + +Run `afl-fuzz` with `-A` option to use CoreSight mode. + +```bash +sudo afl-fuzz -A -i input -o output -- $OUTPUT @@ +``` + +## Environment Variables + +There are AFL++ CoreSight mode-specific environment variables for run-time configuration. + +* `AFL_CS_CUSTOM_BIN` overrides the proxy application path. `afl-cs-proxy` will be used if not defined. + +* `AFLCS_COV` specifies coverage type on CoreSight trace decoding. `edge` and `path` is supported. The default value is `edge`. +* `AFLCS_UDMABUF` is the u-dma-buf device number used to store trace data in the DMA region. The default value is `0`. + +## Acknowledgements + +This project has received funding from the Acquisition, Technology & Logistics Agency (ATLA) under the National Security Technology Research Promotion Fund 2021 (JPJ004596). diff --git a/coresight_mode/coresight-trace b/coresight_mode/coresight-trace new file mode 160000 index 00000000..ec0fd610 --- /dev/null +++ b/coresight_mode/coresight-trace @@ -0,0 +1 @@ +Subproject commit ec0fd6104720ac0b59967616363dc18209adc02e diff --git a/coresight_mode/patchelf b/coresight_mode/patchelf new file mode 160000 index 00000000..7ec8edbe --- /dev/null +++ b/coresight_mode/patchelf @@ -0,0 +1 @@ +Subproject commit 7ec8edbe094ee13c91dadca191f92b9dfac8c0f9 diff --git a/coresight_mode/patches/0001-Add-AFL-forkserver.patch b/coresight_mode/patches/0001-Add-AFL-forkserver.patch new file mode 100644 index 00000000..51c242c4 --- /dev/null +++ b/coresight_mode/patches/0001-Add-AFL-forkserver.patch @@ -0,0 +1,117 @@ +diff --git a/glibc-2.33/elf/rtld.c b/glibc-2.33/elf/rtld.c +index 596b6ac3..2ee270d4 100644 +--- a/glibc-2.33/elf/rtld.c ++++ b/glibc-2.33/elf/rtld.c +@@ -169,6 +169,99 @@ uintptr_t __pointer_chk_guard_local + strong_alias (__pointer_chk_guard_local, __pointer_chk_guard) + #endif + ++#define AFLCS_RTLD 1 ++ ++#if AFLCS_RTLD ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#define FORKSRV_FD 198 ++ ++#define AFLCS_ENABLE "__AFLCS_ENABLE" ++ ++/* We use this additional AFLCS_# AFLCS_#+1 pair to communicate with proxy */ ++#define AFLCS_FORKSRV_FD (FORKSRV_FD - 3) ++#define AFLCS_RTLD_SNIPPET do { __cs_start_forkserver(); } while(0) ++ ++/* Fork server logic, invoked before we return from _dl_start. */ ++ ++static void __cs_start_forkserver(void) { ++ int status; ++ pid_t child_pid; ++ static char tmp[4] = {0, 0, 0, 0}; ++ ++ if (!getenv(AFLCS_ENABLE)) { ++ return; ++ } ++ ++ if (write(AFLCS_FORKSRV_FD + 1, tmp, 4) != 4) { ++ _exit(-1); ++ } ++ ++ /* All right, let's await orders... */ ++ while (1) { ++ /* Whoops, parent dead? */ ++ if (read(AFLCS_FORKSRV_FD, tmp, 4) != 4) { ++ _exit(1); ++ } ++ ++ child_pid = INLINE_SYSCALL(clone, 5, ++ CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID | SIGCHLD, 0, ++ NULL, NULL, &THREAD_SELF->tid); ++ if (child_pid < 0) { ++ _exit(4); ++ } ++ if (!child_pid) { ++ /* Child process. Wait for parent start tracing */ ++ kill(getpid(), SIGSTOP); ++ /* Close descriptors and run free. */ ++ close(AFLCS_FORKSRV_FD); ++ close(AFLCS_FORKSRV_FD + 1); ++ return; ++ } ++ ++ /* Parent. */ ++ if (write(AFLCS_FORKSRV_FD + 1, &child_pid, 4) != 4) { ++ _exit(5); ++ } ++ ++ /* Wait until SIGCONT is signaled. */ ++ if (waitpid(child_pid, &status, WCONTINUED) < 0) { ++ _exit(6); ++ } ++ if (!WIFCONTINUED(status)) { ++ /* Relay status to proxy. */ ++ if (write(AFLCS_FORKSRV_FD + 1, &status, 4) != 4) { ++ _exit(7); ++ } ++ continue; ++ } ++ while (1) { ++ /* Get status. */ ++ if (waitpid(child_pid, &status, WUNTRACED) < 0) { ++ _exit(8); ++ } ++ /* Relay status to proxy. */ ++ if (write(AFLCS_FORKSRV_FD + 1, &status, 4) != 4) { ++ _exit(9); ++ } ++ if (!(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP)) { ++ /* The child process is exited. */ ++ break; ++ } ++ } ++ } ++} ++ ++#endif /* AFLCS_RTLD */ ++ + /* Check that AT_SECURE=0, or that the passed name does not contain + directories and is not overly long. Reject empty names + unconditionally. */ +@@ -588,6 +681,12 @@ _dl_start (void *arg) + # define ELF_MACHINE_START_ADDRESS(map, start) (start) + #endif + ++ /* AFL-CS-START */ ++#if AFLCS_RTLD ++ AFLCS_RTLD_SNIPPET; ++#endif ++ /* AFL-CS-END */ ++ + return ELF_MACHINE_START_ADDRESS (GL(dl_ns)[LM_ID_BASE]._ns_loaded, entry); + } + } diff --git a/include/common.h b/include/common.h index e3997aa4..6c8e3b3a 100644 --- a/include/common.h +++ b/include/common.h @@ -46,6 +46,7 @@ void check_environment_vars(char **env); char **argv_cpy_dup(int argc, char **argv); void argv_cpy_free(char **argv); +char **get_cs_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv); char **get_qemu_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv); char **get_wine_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv); char * get_afl_env(char *env); diff --git a/include/forkserver.h b/include/forkserver.h index 7af01cb2..464f208d 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -82,6 +82,8 @@ typedef struct afl_forkserver { bool frida_asan; /* if running with asan in frida mode */ + bool cs_mode; /* if running in CoreSight mode or not */ + bool use_stdin; /* use stdin for sending data */ bool no_unlink; /* do not unlink cur_input */ diff --git a/src/afl-analyze.c b/src/afl-analyze.c index d52a6d6e..c8b82428 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -77,6 +77,7 @@ static volatile u8 stop_soon; /* Ctrl-C pressed? */ static u8 *target_path; static u8 frida_mode; static u8 qemu_mode; +static u8 cs_mode; static u32 map_size = MAP_SIZE; static afl_forkserver_t fsrv = {0}; /* The forkserver */ @@ -790,6 +791,8 @@ static void set_up_environment(char **argv) { } else { + /* CoreSight mode uses the default behavior. */ + setenv("LD_PRELOAD", getenv("AFL_PRELOAD"), 1); setenv("DYLD_INSERT_LIBRARIES", getenv("AFL_PRELOAD"), 1); @@ -845,6 +848,7 @@ static void usage(u8 *argv0) { " -f file - input file read by the tested program (stdin)\n" " -t msec - timeout for each run (%u ms)\n" " -m megs - memory limit for child process (%u MB)\n" + " -A - use binary-only instrumentation (CoreSight mode)\n" " -O - use binary-only instrumentation (FRIDA mode)\n" " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" @@ -890,7 +894,7 @@ int main(int argc, char **argv_orig, char **envp) { afl_fsrv_init(&fsrv); - while ((opt = getopt(argc, argv, "+i:f:m:t:eOQUWh")) > 0) { + while ((opt = getopt(argc, argv, "+i:f:m:t:eAOQUWh")) > 0) { switch (opt) { @@ -989,6 +993,14 @@ int main(int argc, char **argv_orig, char **envp) { break; + case 'A': /* CoreSight mode */ + + if (cs_mode) { FATAL("Multiple -A options not supported"); } + + cs_mode = 1; + fsrv.cs_mode = cs_mode; + break; + case 'O': /* FRIDA mode */ if (frida_mode) { FATAL("Multiple -O options not supported"); } @@ -1080,6 +1092,10 @@ int main(int argc, char **argv_orig, char **envp) { } + } else if (cs_mode) { + + use_argv = get_cs_argv(argv[0], &target_path, argc - optind, argv + optind); + } else { use_argv = argv + optind; diff --git a/src/afl-common.c b/src/afl-common.c index ec3b2f3f..6c2d0753 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -204,6 +204,35 @@ void argv_cpy_free(char **argv) { } +/* Rewrite argv for CoreSight process tracer. */ + +char **get_cs_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { + + if (unlikely(getenv("AFL_CS_CUSTOM_BIN"))) { + + WARNF( + "AFL_CS_CUSTOM_BIN is enabled. " + "You must run your target under afl-cs-proxy on your own!"); + return argv; + + } + + char **new_argv = ck_alloc(sizeof(char *) * (argc + 4)); + if (unlikely(!new_argv)) { FATAL("Illegal amount of arguments specified"); } + + memcpy(&new_argv[3], &argv[1], (int)(sizeof(char *)) * (argc - 1)); + new_argv[argc + 3] = NULL; + + new_argv[2] = *target_path_p; + new_argv[1] = "--"; + + /* Now we need to actually find the cs-proxy binary to put in argv[0]. */ + + *target_path_p = new_argv[0] = find_afl_binary(own_loc, "afl-cs-proxy"); + return new_argv; + +} + /* Rewrite argv for QEMU. */ char **get_qemu_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 9262d718..e5a4d3d1 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -2645,6 +2645,7 @@ void check_binary(afl_state_t *afl, u8 *fname) { if (afl->afl_env.afl_skip_bin_check || afl->use_wine || afl->unicorn_mode || (afl->fsrv.qemu_mode && getenv("AFL_QEMU_CUSTOM_BIN")) || + (afl->fsrv.cs_mode && getenv("AFL_CS_CUSTOM_BIN")) || afl->non_instrumented_mode) { return; @@ -2721,7 +2722,7 @@ void check_binary(afl_state_t *afl, u8 *fname) { #endif /* ^!__APPLE__ */ if (!afl->fsrv.qemu_mode && !afl->fsrv.frida_mode && !afl->unicorn_mode && - !afl->non_instrumented_mode && + !afl->fsrv.cs_mode && !afl->non_instrumented_mode && !memmem(f_data, f_len, SHM_ENV_VAR, strlen(SHM_ENV_VAR) + 1)) { SAYF("\n" cLRD "[-] " cRST @@ -2752,7 +2753,7 @@ void check_binary(afl_state_t *afl, u8 *fname) { } - if ((afl->fsrv.qemu_mode || afl->fsrv.frida_mode) && + if ((afl->fsrv.cs_mode || afl->fsrv.qemu_mode || afl->fsrv.frida_mode) && memmem(f_data, f_len, SHM_ENV_VAR, strlen(SHM_ENV_VAR) + 1)) { SAYF("\n" cLRD "[-] " cRST diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 808bf258..426580d2 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -285,7 +285,7 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, "afl_banner : %s\n" "afl_version : " VERSION "\n" - "target_mode : %s%s%s%s%s%s%s%s%s\n" + "target_mode : %s%s%s%s%s%s%s%s%s%s\n" "command_line : %s\n", (afl->start_time - afl->prev_run_time) / 1000, cur_time / 1000, (afl->prev_run_time + cur_time - afl->start_time) / 1000, @@ -321,12 +321,13 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, afl->q_testcase_cache_count, afl->q_testcase_evictions, afl->use_banner, afl->unicorn_mode ? "unicorn" : "", afl->fsrv.qemu_mode ? "qemu " : "", + afl->fsrv.cs_mode ? "coresight" : "", afl->non_instrumented_mode ? " non_instrumented " : "", afl->no_forkserver ? "no_fsrv " : "", afl->crash_mode ? "crash " : "", afl->persistent_mode ? "persistent " : "", afl->shmem_testcase_mode ? "shmem_testcase " : "", afl->deferred_mode ? "deferred " : "", - (afl->unicorn_mode || afl->fsrv.qemu_mode || + (afl->unicorn_mode || afl->fsrv.qemu_mode || afl->fsrv.cs_mode || afl->non_instrumented_mode || afl->no_forkserver || afl->crash_mode || afl->persistent_mode || afl->deferred_mode) ? "" @@ -1238,7 +1239,9 @@ void show_init_stats(afl_state_t *afl) { // SAYF("\n"); - if (avg_us > ((afl->fsrv.qemu_mode || afl->unicorn_mode) ? 50000 : 10000)) { + if (avg_us > ((afl->fsrv.cs_mode || afl->fsrv.qemu_mode || afl->unicorn_mode) + ? 50000 + : 10000)) { WARNF(cLRD "The target binary is pretty slow! See %s/perf_tips.md.", doc_path); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index c08b8fbb..99eebfaa 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -113,6 +113,7 @@ static void usage(u8 *argv0, int more_help) { "maximum.\n" " -m megs - memory limit for child process (%u MB, 0 = no limit " "[default])\n" + " -A - use binary-only instrumentation (CoreSight mode)\n" " -O - use binary-only instrumentation (FRIDA mode)\n" " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" @@ -434,7 +435,8 @@ int main(int argc, char **argv_orig, char **envp) { while ((opt = getopt( argc, argv, - "+b:B:c:CdDe:E:hi:I:f:F:l:L:m:M:nNOo:p:RQs:S:t:T:UV:Wx:Z")) > 0) { + "+Ab:B:c:CdDe:E:hi:I:f:F:l:L:m:M:nNOo:p:RQs:S:t:T:UV:Wx:Z")) > + 0) { switch (opt) { @@ -825,6 +827,13 @@ int main(int argc, char **argv_orig, char **envp) { afl->use_banner = optarg; break; + case 'A': /* CoreSight mode */ + + if (afl->fsrv.cs_mode) { FATAL("Multiple -A options not supported"); } + afl->fsrv.cs_mode = 1; + + break; + case 'O': /* FRIDA mode */ if (afl->fsrv.frida_mode) { @@ -1212,6 +1221,7 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->crash_mode) { FATAL("-C and -n are mutually exclusive"); } if (afl->fsrv.frida_mode) { FATAL("-O and -n are mutually exclusive"); } if (afl->fsrv.qemu_mode) { FATAL("-Q and -n are mutually exclusive"); } + if (afl->fsrv.cs_mode) { FATAL("-A and -n are mutually exclusive"); } if (afl->unicorn_mode) { FATAL("-U and -n are mutually exclusive"); } } @@ -1458,6 +1468,8 @@ int main(int argc, char **argv_orig, char **envp) { } else { + /* CoreSight mode uses the default behavior. */ + setenv("LD_PRELOAD", getenv("AFL_PRELOAD"), 1); setenv("DYLD_INSERT_LIBRARIES", getenv("AFL_PRELOAD"), 1); @@ -1651,7 +1663,7 @@ int main(int argc, char **argv_orig, char **envp) { } - if (!afl->fsrv.qemu_mode && !afl->fsrv.frida_mode && + if (!afl->fsrv.qemu_mode && !afl->fsrv.frida_mode && !afl->fsrv.cs_mode && !afl->non_instrumented_mode) { check_binary(afl, afl->cmplog_binary); @@ -1697,6 +1709,11 @@ int main(int argc, char **argv_orig, char **envp) { } + } else if (afl->fsrv.cs_mode) { + + use_argv = get_cs_argv(argv[0], &afl->fsrv.target_path, argc - optind, + argv + optind); + } else { use_argv = argv + optind; @@ -1704,7 +1721,7 @@ int main(int argc, char **argv_orig, char **envp) { } if (afl->non_instrumented_mode || afl->fsrv.qemu_mode || - afl->fsrv.frida_mode || afl->unicorn_mode) { + afl->fsrv.frida_mode || afl->fsrv.cs_mode || afl->unicorn_mode) { map_size = afl->fsrv.real_map_size = afl->fsrv.map_size = MAP_SIZE; afl->virgin_bits = ck_realloc(afl->virgin_bits, map_size); @@ -1724,7 +1741,7 @@ int main(int argc, char **argv_orig, char **envp) { afl_shm_init(&afl->shm, afl->fsrv.map_size, afl->non_instrumented_mode); if (!afl->non_instrumented_mode && !afl->fsrv.qemu_mode && - !afl->unicorn_mode && !afl->fsrv.frida_mode && + !afl->unicorn_mode && !afl->fsrv.frida_mode && !afl->fsrv.cs_mode && !afl->afl_env.afl_skip_bin_check) { if (map_size <= DEFAULT_SHMEM_SIZE) { @@ -1777,6 +1794,7 @@ int main(int argc, char **argv_orig, char **envp) { afl_fsrv_init_dup(&afl->cmplog_fsrv, &afl->fsrv); // TODO: this is semi-nice afl->cmplog_fsrv.trace_bits = afl->fsrv.trace_bits; + afl->cmplog_fsrv.cs_mode = afl->fsrv.cs_mode; afl->cmplog_fsrv.qemu_mode = afl->fsrv.qemu_mode; afl->cmplog_fsrv.frida_mode = afl->fsrv.frida_mode; afl->cmplog_fsrv.cmplog_binary = afl->cmplog_binary; @@ -1785,7 +1803,7 @@ int main(int argc, char **argv_orig, char **envp) { if ((map_size <= DEFAULT_SHMEM_SIZE || afl->cmplog_fsrv.map_size < map_size) && !afl->non_instrumented_mode && !afl->fsrv.qemu_mode && - !afl->fsrv.frida_mode && !afl->unicorn_mode && + !afl->fsrv.frida_mode && !afl->unicorn_mode && !afl->fsrv.cs_mode && !afl->afl_env.afl_skip_bin_check) { afl->cmplog_fsrv.map_size = MAX(map_size, (u32)DEFAULT_SHMEM_SIZE); diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 6c06c476..daaed767 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -690,6 +690,8 @@ static void set_up_environment(afl_forkserver_t *fsrv, char **argv) { } else { + /* CoreSight mode uses the default behavior. */ + setenv("LD_PRELOAD", getenv("AFL_PRELOAD"), 1); setenv("DYLD_INSERT_LIBRARIES", getenv("AFL_PRELOAD"), 1); @@ -843,6 +845,7 @@ static void usage(u8 *argv0) { " -t msec - timeout for each run (none)\n" " -m megs - memory limit for child process (%u MB)\n" " -O - use binary-only instrumentation (FRIDA mode)\n" + " -P - use binary-only instrumentation (CoreSight mode)\n" " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use Unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine mode)\n" @@ -917,7 +920,7 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_QUIET") != NULL) { be_quiet = true; } - while ((opt = getopt(argc, argv, "+i:o:f:m:t:A:eqCZOQUWbcrsh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:A:eqCZOPQUWbcrsh")) > 0) { switch (opt) { @@ -1060,6 +1063,15 @@ int main(int argc, char **argv_orig, char **envp) { break; + /* FIXME: We want to use -P for consistency, but it is already unsed for + * undocumenetd feature "Another afl-cmin specific feature." */ + case 'P': /* CoreSight mode */ + + if (fsrv->cs_mode) { FATAL("Multiple -P options not supported"); } + + fsrv->cs_mode = true; + break; + case 'Q': if (fsrv->qemu_mode) { FATAL("Multiple -Q options not supported"); } @@ -1124,6 +1136,7 @@ int main(int argc, char **argv_orig, char **envp) { } + if (fsrv->cs_mode && !mem_limit_given) { fsrv->mem_limit = MEM_LIMIT; } if (fsrv->qemu_mode && !mem_limit_given) { fsrv->mem_limit = MEM_LIMIT_QEMU; } if (unicorn_mode && !mem_limit_given) { fsrv->mem_limit = MEM_LIMIT_UNICORN; } @@ -1204,6 +1217,11 @@ int main(int argc, char **argv_orig, char **envp) { } + } else if (fsrv->cs_mode) { + + use_argv = + get_cs_argv(argv[0], &fsrv->target_path, argc - optind, argv + optind); + } else { use_argv = argv + optind; @@ -1230,7 +1248,7 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->shmem_fuzz_len = (u32 *)map; fsrv->shmem_fuzz = map + sizeof(u32); - if (!fsrv->qemu_mode && !unicorn_mode) { + if (!fsrv->cs_mode && !fsrv->qemu_mode && !unicorn_mode) { u32 save_be_quiet = be_quiet; be_quiet = !debug; diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 3f6f14f9..212b6251 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -808,6 +808,8 @@ static void set_up_environment(afl_forkserver_t *fsrv, char **argv) { } else { + /* CoreSight mode uses the default behavior. */ + setenv("LD_PRELOAD", getenv("AFL_PRELOAD"), 1); setenv("DYLD_INSERT_LIBRARIES", getenv("AFL_PRELOAD"), 1); @@ -921,7 +923,7 @@ int main(int argc, char **argv_orig, char **envp) { SAYF(cCYA "afl-tmin" VERSION cRST " by Michal Zalewski\n"); - while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeOQUWHh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeAOQUWHh")) > 0) { switch (opt) { @@ -1033,6 +1035,13 @@ int main(int argc, char **argv_orig, char **envp) { break; + case 'A': /* CoreSight mode */ + + if (fsrv->cs_mode) { FATAL("Multiple -A options not supported"); } + + fsrv->cs_mode = 1; + break; + case 'O': /* FRIDA mode */ if (fsrv->frida_mode) { FATAL("Multiple -O options not supported"); } @@ -1152,6 +1161,11 @@ int main(int argc, char **argv_orig, char **envp) { } + } else if (fsrv->cs_mode) { + + use_argv = + get_cs_argv(argv[0], &fsrv->target_path, argc - optind, argv + optind); + } else { use_argv = argv + optind; -- cgit 1.4.1 From d63d69a1f66e00f453e358662527fbd78361147d Mon Sep 17 00:00:00 2001 From: Akira Moroo Date: Fri, 12 Nov 2021 05:33:40 +0000 Subject: Clarify usage message for ARM CoreSight mode REF: https://github.com/AFLplusplus/AFLplusplus/pull/1156#issuecomment-966196217 Signed-off-by: Akira Moroo --- src/afl-analyze.c | 2 +- src/afl-fuzz.c | 2 +- src/afl-showmap.c | 2 +- src/afl-tmin.c | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/afl-analyze.c b/src/afl-analyze.c index c8b82428..4872c60d 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -848,7 +848,7 @@ static void usage(u8 *argv0) { " -f file - input file read by the tested program (stdin)\n" " -t msec - timeout for each run (%u ms)\n" " -m megs - memory limit for child process (%u MB)\n" - " -A - use binary-only instrumentation (CoreSight mode)\n" + " -A - use binary-only instrumentation (ARM CoreSight mode)\n" " -O - use binary-only instrumentation (FRIDA mode)\n" " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 99eebfaa..6538e0a0 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -113,7 +113,7 @@ static void usage(u8 *argv0, int more_help) { "maximum.\n" " -m megs - memory limit for child process (%u MB, 0 = no limit " "[default])\n" - " -A - use binary-only instrumentation (CoreSight mode)\n" + " -A - use binary-only instrumentation (ARM CoreSight mode)\n" " -O - use binary-only instrumentation (FRIDA mode)\n" " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" diff --git a/src/afl-showmap.c b/src/afl-showmap.c index daaed767..4c207d62 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -845,7 +845,7 @@ static void usage(u8 *argv0) { " -t msec - timeout for each run (none)\n" " -m megs - memory limit for child process (%u MB)\n" " -O - use binary-only instrumentation (FRIDA mode)\n" - " -P - use binary-only instrumentation (CoreSight mode)\n" + " -P - use binary-only instrumentation (ARM CoreSight mode)\n" " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use Unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine mode)\n" diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 212b6251..42883404 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -866,6 +866,7 @@ static void usage(u8 *argv0) { " -f file - input file read by the tested program (stdin)\n" " -t msec - timeout for each run (%u ms)\n" " -m megs - memory limit for child process (%u MB)\n" + " -A - use binary-only instrumentation (ARM CoreSight mode)\n" " -O - use binary-only instrumentation (FRIDA mode)\n" " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" -- cgit 1.4.1 From cf0fd0ff3342fe1b9a028dca95b750f730e2afd6 Mon Sep 17 00:00:00 2001 From: Akira Moroo Date: Fri, 12 Nov 2021 05:34:39 +0000 Subject: Remove unnecessary mem_limit in afl-showmap Signed-off-by: Akira Moroo --- src/afl-showmap.c | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 4c207d62..57737562 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1136,7 +1136,6 @@ int main(int argc, char **argv_orig, char **envp) { } - if (fsrv->cs_mode && !mem_limit_given) { fsrv->mem_limit = MEM_LIMIT; } if (fsrv->qemu_mode && !mem_limit_given) { fsrv->mem_limit = MEM_LIMIT_QEMU; } if (unicorn_mode && !mem_limit_given) { fsrv->mem_limit = MEM_LIMIT_UNICORN; } -- cgit 1.4.1 From c2feee4ed1b35cc590e2beaa595d710b09e1427c Mon Sep 17 00:00:00 2001 From: Akira Moroo Date: Fri, 12 Nov 2021 06:01:19 +0000 Subject: Add platform check for `-A` CoreSight mode REF: https://github.com/AFLplusplus/AFLplusplus/pull/1156#discussion_r747454306 Signed-off-by: Akira Moroo --- src/afl-analyze.c | 6 ++++++ src/afl-fuzz.c | 7 +++++++ src/afl-showmap.c | 6 ++++++ src/afl-tmin.c | 6 ++++++ 4 files changed, 25 insertions(+) (limited to 'src') diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 4872c60d..6e1a9e7b 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -997,6 +997,12 @@ int main(int argc, char **argv_orig, char **envp) { if (cs_mode) { FATAL("Multiple -A options not supported"); } + if (!(__aarch64__ && __linux__)) { + + FATAL("-A option is not supported on this platform"); + + } + cs_mode = 1; fsrv.cs_mode = cs_mode; break; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 6538e0a0..e6d9508b 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -830,6 +830,13 @@ int main(int argc, char **argv_orig, char **envp) { case 'A': /* CoreSight mode */ if (afl->fsrv.cs_mode) { FATAL("Multiple -A options not supported"); } + + if (!(__aarch64__ && __linux__)) { + + FATAL("-A option is not supported on this platform"); + + } + afl->fsrv.cs_mode = 1; break; diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 57737562..0e7d9df6 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1069,6 +1069,12 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->cs_mode) { FATAL("Multiple -P options not supported"); } + if (!(__aarch64__ && __linux__)) { + + FATAL("-P option is not supported on this platform"); + + } + fsrv->cs_mode = true; break; diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 42883404..110beed5 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -1040,6 +1040,12 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->cs_mode) { FATAL("Multiple -A options not supported"); } + if (!(__aarch64__ && __linux__)) { + + FATAL("-A option is not supported on this platform"); + + } + fsrv->cs_mode = 1; break; -- cgit 1.4.1 From feff8191ecbde9bb039e2311440f47e8f0325413 Mon Sep 17 00:00:00 2001 From: Akira Moroo Date: Mon, 15 Nov 2021 01:57:36 +0000 Subject: Fix platform check for `-A` CoreSight mode Signed-off-by: Akira Moroo --- src/afl-analyze.c | 10 ++++------ src/afl-fuzz.c | 10 ++++------ src/afl-showmap.c | 10 ++++------ src/afl-tmin.c | 10 ++++------ 4 files changed, 16 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 6e1a9e7b..bc562c15 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -995,13 +995,11 @@ int main(int argc, char **argv_orig, char **envp) { case 'A': /* CoreSight mode */ - if (cs_mode) { FATAL("Multiple -A options not supported"); } - - if (!(__aarch64__ && __linux__)) { - - FATAL("-A option is not supported on this platform"); +#if !defined(__aarch64__) || !defined(__linux__) + FATAL("-A option is not supported on this platform"); +#endif - } + if (cs_mode) { FATAL("Multiple -A options not supported"); } cs_mode = 1; fsrv.cs_mode = cs_mode; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index e6d9508b..44363c2b 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -829,13 +829,11 @@ int main(int argc, char **argv_orig, char **envp) { case 'A': /* CoreSight mode */ - if (afl->fsrv.cs_mode) { FATAL("Multiple -A options not supported"); } - - if (!(__aarch64__ && __linux__)) { - - FATAL("-A option is not supported on this platform"); + #if !defined(__aarch64__) || !defined(__linux__) + FATAL("-A option is not supported on this platform"); + #endif - } + if (afl->fsrv.cs_mode) { FATAL("Multiple -A options not supported"); } afl->fsrv.cs_mode = 1; diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 0e7d9df6..899baaa0 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1067,13 +1067,11 @@ int main(int argc, char **argv_orig, char **envp) { * undocumenetd feature "Another afl-cmin specific feature." */ case 'P': /* CoreSight mode */ - if (fsrv->cs_mode) { FATAL("Multiple -P options not supported"); } - - if (!(__aarch64__ && __linux__)) { - - FATAL("-P option is not supported on this platform"); +#if !defined(__aarch64__) || !defined(__linux__) + FATAL("-P option is not supported on this platform"); +#endif - } + if (fsrv->cs_mode) { FATAL("Multiple -P options not supported"); } fsrv->cs_mode = true; break; diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 110beed5..22383a4e 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -1038,13 +1038,11 @@ int main(int argc, char **argv_orig, char **envp) { case 'A': /* CoreSight mode */ - if (fsrv->cs_mode) { FATAL("Multiple -A options not supported"); } - - if (!(__aarch64__ && __linux__)) { - - FATAL("-A option is not supported on this platform"); +#if !defined(__aarch64__) || !defined(__linux__) + FATAL("-A option is not supported on this platform"); +#endif - } + if (fsrv->cs_mode) { FATAL("Multiple -A options not supported"); } fsrv->cs_mode = 1; break; -- cgit 1.4.1 From d4a0fd41cd29b5862cac0b99b96b2afc67e33c7d Mon Sep 17 00:00:00 2001 From: Akira Moroo Date: Mon, 15 Nov 2021 02:15:58 +0000 Subject: Check `-M` / `-S` is not specified with `-A` Signed-off-by: Akira Moroo --- src/afl-fuzz.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'src') diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 44363c2b..dfd62db8 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -565,6 +565,12 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->fsrv.cs_mode) { + + FATAL("-M is not supported in ARM CoreSight mode"); + + } + if (afl->sync_id) { FATAL("Multiple -S or -M options not supported"); } /* sanity check for argument: should not begin with '-' (possible @@ -611,6 +617,12 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->fsrv.cs_mode) { + + FATAL("-S is not supported in ARM CoreSight mode"); + + } + if (afl->sync_id) { FATAL("Multiple -S or -M options not supported"); } /* sanity check for argument: should not begin with '-' (possible @@ -833,6 +845,12 @@ int main(int argc, char **argv_orig, char **envp) { FATAL("-A option is not supported on this platform"); #endif + if (afl->is_main_node || afl->is_secondary_node) { + + FATAL("ARM CoreSight mode is not supported with -M / -S"); + + } + if (afl->fsrv.cs_mode) { FATAL("Multiple -A options not supported"); } afl->fsrv.cs_mode = 1; -- cgit 1.4.1 From 268339a683aab00f8487eac1ca31ef5d6c6abc4b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 16 Nov 2021 11:03:53 +0100 Subject: showmap -A -> -H, accurate help output --- GNUmakefile | 2 +- afl-cmin | 6 +++--- afl-cmin.bash | 4 ++-- coresight_mode/README.md | 1 - docs/Changelog.md | 6 +++++- src/afl-analyze.c | 7 ++++++- src/afl-fuzz-bitmap.c | 6 ++---- src/afl-fuzz.c | 7 ++++++- src/afl-showmap.c | 19 ++++++++++++------- src/afl-tmin.c | 7 ++++++- 10 files changed, 43 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/GNUmakefile b/GNUmakefile index ab57e7ad..673d2bf8 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -346,7 +346,7 @@ help: @echo "HELP --- the following make targets exist:" @echo "==========================================" @echo "all: just the main afl++ binaries" - @echo "binary-only: everything for binary-only fuzzing: coresight_mode, qemu_mode, unicorn_mode, libdislocator, libtokencap" + @echo "binary-only: everything for binary-only fuzzing: qemu_mode, frida_mode, unicorn_mode, coresight_mode, libdislocator, libtokencap" @echo "source-only: everything for source code fuzzing: gcc_plugin, libdislocator, libtokencap" @echo "distrib: everything (for both binary-only and source code fuzzing)" @echo "man: creates simple man pages from the help option of the programs" diff --git a/afl-cmin b/afl-cmin index e6f8c175..879aead2 100755 --- a/afl-cmin +++ b/afl-cmin @@ -396,7 +396,7 @@ BEGIN { system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") } else { system("cp \""in_dir"/"first_file"\" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" . ### Version ++3.15a (dev) - documentation restructuring, made possible by Google Season of Docs :) + - new binary-only fuzzing mode: coresight_mode for aarch64 CPUs :) + thanks to RICSecLab submitting! - afl-fuzz: - cmplog binaries will need to be recompiled for this version (it is better!) - fix a regression introduced in 3.10 that resulted in less coverage being detected. thanks to Collin May for reporting! - - added AFL_IGNORE_PROBLEMS plus checks to identify and abort on + - added AFL_IGNORE_PROBLEMS, plus checks to identify and abort on incorrect LTO usage setups and enhanced the READMEs for better information on how to deal with instrumenting libraries - fix -n dumb mode (nobody should use this) @@ -30,6 +32,8 @@ sending a mail to . - honor persistent mode for more speed. thanks to dloffre-snl for reporting! - fix bug where targets are not killed on timeouts + - moved hidden afl-showmap -A option to -H to be used for + coresight_mode - Prevent accidently killing non-afl/fuzz services when aborting afl-showmap and other tools. - afl-cc: diff --git a/src/afl-analyze.c b/src/afl-analyze.c index bc562c15..ac5a324c 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -848,12 +848,17 @@ static void usage(u8 *argv0) { " -f file - input file read by the tested program (stdin)\n" " -t msec - timeout for each run (%u ms)\n" " -m megs - memory limit for child process (%u MB)\n" +#if defined(__linux__) && defined(__aarch64__) " -A - use binary-only instrumentation (ARM CoreSight mode)\n" +#endif " -O - use binary-only instrumentation (FRIDA mode)\n" +#if defined(__linux__) " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine " - "mode)\n\n" + "mode)\n" +#endif + "\n" "Analysis settings:\n" diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index f7b59f25..a204e374 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -452,14 +452,12 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { if (unlikely(len == 0)) { return 0; } + u8 fn[PATH_MAX]; u8 *queue_fn = ""; - u8 new_bits = '\0'; + u8 new_bits = 0, keeping = 0, res, classified = 0; s32 fd; - u8 keeping = 0, res, classified = 0; u64 cksum = 0; - u8 fn[PATH_MAX]; - /* Update path frequency. */ /* Generating a hash on every input is super expensive. Bad idea and should diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index dfd62db8..195366bd 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -113,12 +113,17 @@ static void usage(u8 *argv0, int more_help) { "maximum.\n" " -m megs - memory limit for child process (%u MB, 0 = no limit " "[default])\n" +#if defined(__linux__) && defined(__aarch64__) " -A - use binary-only instrumentation (ARM CoreSight mode)\n" +#endif " -O - use binary-only instrumentation (FRIDA mode)\n" +#if defined(__linux__) " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine " - "mode)\n\n" + "mode)\n" +#endif + "\n" "Mutator settings:\n" " -D - enable deterministic fuzzing (once per queue entry)\n" diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 899baaa0..0ba265ab 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -844,13 +844,18 @@ static void usage(u8 *argv0) { "Execution control settings:\n" " -t msec - timeout for each run (none)\n" " -m megs - memory limit for child process (%u MB)\n" +#if defined(__linux__) && defined(__aarch64__) + " -A - use binary-only instrumentation (ARM CoreSight mode)\n" +#endif " -O - use binary-only instrumentation (FRIDA mode)\n" - " -P - use binary-only instrumentation (ARM CoreSight mode)\n" +#if defined(__linux__) " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use Unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine mode)\n" " (Not necessary, here for consistency with other afl-* " - "tools)\n\n" + "tools)\n" +#endif + "\n" "Other settings:\n" " -i dir - process all files below this directory, must be combined " "with -o.\n" @@ -920,7 +925,7 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_QUIET") != NULL) { be_quiet = true; } - while ((opt = getopt(argc, argv, "+i:o:f:m:t:A:eqCZOPQUWbcrsh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:A:eqCZOHQUWbcrsh")) > 0) { switch (opt) { @@ -1049,7 +1054,7 @@ int main(int argc, char **argv_orig, char **envp) { quiet_mode = true; break; - case 'A': + case 'H': /* Another afl-cmin specific feature. */ at_file = optarg; break; @@ -1065,13 +1070,13 @@ int main(int argc, char **argv_orig, char **envp) { /* FIXME: We want to use -P for consistency, but it is already unsed for * undocumenetd feature "Another afl-cmin specific feature." */ - case 'P': /* CoreSight mode */ + case 'A': /* CoreSight mode */ #if !defined(__aarch64__) || !defined(__linux__) - FATAL("-P option is not supported on this platform"); + FATAL("-A option is not supported on this platform"); #endif - if (fsrv->cs_mode) { FATAL("Multiple -P options not supported"); } + if (fsrv->cs_mode) { FATAL("Multiple -A options not supported"); } fsrv->cs_mode = true; break; diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 22383a4e..89546c45 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -866,14 +866,19 @@ static void usage(u8 *argv0) { " -f file - input file read by the tested program (stdin)\n" " -t msec - timeout for each run (%u ms)\n" " -m megs - memory limit for child process (%u MB)\n" +#if defined(__linux__) && defined(__aarch64__) " -A - use binary-only instrumentation (ARM CoreSight mode)\n" +#endif " -O - use binary-only instrumentation (FRIDA mode)\n" +#if defined(__linux__) " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine " "mode)\n" " (Not necessary, here for consistency with other afl-* " - "tools)\n\n" + "tools)\n" +#endif + "\n" "Minimization settings:\n" -- cgit 1.4.1 From 1a8d3f82f289d248aa70ce6877b66c7387f77fc9 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 16 Nov 2021 13:01:02 +0100 Subject: fix afl-showmap --- src/afl-showmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 0ba265ab..236553ce 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -925,7 +925,7 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_QUIET") != NULL) { be_quiet = true; } - while ((opt = getopt(argc, argv, "+i:o:f:m:t:A:eqCZOHQUWbcrsh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:AeqCZOH:QUWbcrsh")) > 0) { switch (opt) { -- cgit 1.4.1 From 6f5ba59d99056fabace9a4f1ef8f7706ad3b9386 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 Nov 2021 19:18:26 +0100 Subject: further fixes for new pass manager --- src/afl-cc.c | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/afl-cc.c b/src/afl-cc.c index 58d978ea..ca413417 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -462,7 +462,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ +#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); @@ -473,6 +473,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { cc_params[cc_par_cnt++] = alloc_printf("%s/split-switches-pass.so", obj_path); #endif + } } @@ -487,10 +488,10 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_VERSION_MAJOR >= 7 /* use new pass manager */ +#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/compare-transform-pass.so", obj_path); + cc_params[cc_par_cnt++] = alloc_printf( + "-fpass-plugin=%s/compare-transform-pass.so", obj_path); #else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; @@ -513,7 +514,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_MAJOR >= 7 +#if LLVM_MAJOR >= 11 cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/split-compares-pass.so", obj_path); @@ -554,7 +555,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { alloc_printf("%s/cmplog-switches-pass.so", obj_path); // reuse split switches from laf -#if LLVM_MAJOR >= 7 +#if LLVM_MAJOR >= 11 cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); @@ -590,15 +591,8 @@ static void edit_params(u32 argc, char **argv, char **envp) { free(ld_path); cc_params[cc_par_cnt++] = "-Wl,--allow-multiple-definition"; - - if (instrument_mode == INSTRUMENT_CFG || - instrument_mode == INSTRUMENT_PCGUARD) - cc_params[cc_par_cnt++] = alloc_printf( - "-Wl,-mllvm=-load=%s/SanitizerCoverageLTO.so", obj_path); - else - - cc_params[cc_par_cnt++] = alloc_printf( - "-Wl,-mllvm=-load=%s/afl-llvm-lto-instrumentation.so", obj_path); + cc_params[cc_par_cnt++] = + alloc_printf("-Wl,-mllvm=-load=%s/SanitizerCoverageLTO.so", obj_path); cc_params[cc_par_cnt++] = lto_flag; } else { @@ -654,15 +648,17 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_MAJOR >= 7 +#if LLVM_MAJOR >= 11 cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/afl-llvm-pass.so", obj_path); + cc_params[cc_par_cnt++] = + alloc_printf("-fpass-plugin=%s/afl-llvm-pass.so", obj_path); #else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); #endif + } } -- cgit 1.4.1 From 1fbb7cb2fa0ad10c09f8515124989d119558121d Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Tue, 23 Nov 2021 21:07:11 +0100 Subject: fix switch to new pass manager --- src/afl-cc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/afl-cc.c b/src/afl-cc.c index ca413417..9173fc4b 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -462,7 +462,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ +#if LLVM_MAJOR >= 11 /* use new pass manager */ cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); @@ -488,7 +488,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ +#if LLVM_MAJOR >= 11 /* use new pass manager */ cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; cc_params[cc_par_cnt++] = alloc_printf( "-fpass-plugin=%s/compare-transform-pass.so", obj_path); -- cgit 1.4.1 From 91f1c17c8f00a40d3e975b0a6e8c98677acef4a3 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Wed, 24 Nov 2021 00:05:15 +0100 Subject: transform cmplog-instructions pass to new pass manager --- instrumentation/cmplog-instructions-pass.cc | 37 ++++++++++++++++++++++++++--- src/afl-cc.c | 11 +++++---- 2 files changed, 41 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index a7b7aac8..f2f0de82 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -28,11 +28,16 @@ #include "llvm/Config/llvm-config.h" #include "llvm/ADT/Statistic.h" #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#if LLVM_MAJOR >= 11 +// #include "llvm/Passes/PassPlugin.h" +// #include "llvm/Passes/PassBuilder.h" + #include "llvm/IR/PassManager.h" +#else + #include "llvm/IR/LegacyPassManager.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" @@ -54,6 +59,15 @@ using namespace llvm; namespace { +#if LLVM_MAJOR >= 11 /* use new pass manager */ +class CmpLogInstructions : public PassInfoMixin { + public: + CmpLogInstructions() { + + initInstrumentList(); + + } +#else class CmpLogInstructions : public ModulePass { public: @@ -63,7 +77,11 @@ class CmpLogInstructions : public ModulePass { initInstrumentList(); } +#endif +#if LLVM_MAJOR >= 11 /* use new pass manager */ + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +#else bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR < 4 @@ -76,6 +94,7 @@ class CmpLogInstructions : public ModulePass { return "cmplog instructions"; } +#endif private: bool hookInstrs(Module &M); @@ -84,7 +103,9 @@ class CmpLogInstructions : public ModulePass { } // namespace +#if LLVM_MAJOR <= 10 /* use old pass manager */ char CmpLogInstructions::ID = 0; +#endif template Iterator Unique(Iterator first, Iterator last) { @@ -567,7 +588,12 @@ bool CmpLogInstructions::hookInstrs(Module &M) { } +#if LLVM_MAJOR >= 11 /* use new pass manager */ +PreservedAnalyses CmpLogInstructions::run(Module & M, + ModuleAnalysisManager &MAM) { +#else bool CmpLogInstructions::runOnModule(Module &M) { +#endif if (getenv("AFL_QUIET") == NULL) printf("Running cmplog-instructions-pass by andreafioraldi@gmail.com\n"); @@ -576,10 +602,15 @@ bool CmpLogInstructions::runOnModule(Module &M) { hookInstrs(M); verifyModule(M); +#if LLVM_MAJOR >= 11 /* use new pass manager */ + return PreservedAnalyses::all(); +#else return true; +#endif } +#if LLVM_MAJOR < 11 /* use old pass manager */ static void registerCmpLogInstructionsPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -599,4 +630,4 @@ static RegisterStandardPasses RegisterCmpLogInstructionsPassLTO( PassManagerBuilder::EP_FullLinkTimeOptimizationLast, registerCmpLogInstructionsPass); #endif - +#endif diff --git a/src/afl-cc.c b/src/afl-cc.c index 9173fc4b..c70f193c 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -573,10 +573,6 @@ static void edit_params(u32 argc, char **argv, char **envp) { } -#if LLVM_MAJOR >= 13 - // fuck you llvm 13 - cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; -#endif if (lto_mode && !have_c) { @@ -616,6 +612,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { + cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; @@ -674,11 +671,17 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { +#if LLVM_MAJOR >= 11 + cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; + cc_params[cc_par_cnt++] = + alloc_printf("-fpass-plugin=%s/cmplog-instructions-pass.so", obj_path); +#else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/cmplog-instructions-pass.so", obj_path); +#endif cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; -- cgit 1.4.1 From 55ed2a443c5c61baba37415d4087164454d8a2a8 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 26 Nov 2021 15:30:46 +0100 Subject: remove new llvm pass manager :( --- instrumentation/afl-llvm-dict2file.so.cc | 11 +-- instrumentation/afl-llvm-pass.so.cc | 91 +----------------- instrumentation/cmplog-instructions-pass.cc | 37 +------- instrumentation/compare-transform-pass.so.cc | 116 ++++------------------- instrumentation/split-compares-pass.so.cc | 132 +++------------------------ instrumentation/split-switches-pass.so.cc | 107 ++-------------------- qemu_mode/qemuafl | 2 +- src/afl-cc.c | 42 +-------- 8 files changed, 53 insertions(+), 485 deletions(-) (limited to 'src') diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index 2ffa8fbb..bf07a154 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -589,6 +589,7 @@ bool AFLdict2filePass::runOnModule(Module &M) { if (optLen < 2) { continue; } if (literalLength + 1 == optLen) { // add null byte + thestring.append("\0", 1); } @@ -611,17 +612,11 @@ bool AFLdict2filePass::runOnModule(Module &M) { } */ - - if (!isStdString) { + if (!isStdString && thestring.find('\0', 0) != std::string::npos) { // ensure we do not have garbage size_t offset = thestring.find('\0', 0); - if (offset && offset < optLen && offset + 1 < optLen) { - - optLen = offset + 1; - - } - + if (offset + 1 < optLen) optLen = offset + 1; thestring = thestring.substr(0, optLen); } diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index 9430644e..21ce0cf9 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -45,21 +45,12 @@ typedef long double max_align_t; #endif #include "llvm/IR/IRBuilder.h" -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - #include "llvm/Passes/PassPlugin.h" - #include "llvm/Passes/PassBuilder.h" - #include "llvm/IR/PassManager.h" -#else - #include "llvm/IR/LegacyPassManager.h" - #include "llvm/Transforms/IPO/PassManagerBuilder.h" -#endif +#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" -#if LLVM_VERSION_MAJOR >= 14 /* how about stable interfaces? */ - #include "llvm/Passes/OptimizationLevel.h" -#endif +#include "llvm/Transforms/IPO/PassManagerBuilder.h" #if LLVM_VERSION_MAJOR > 3 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) @@ -77,30 +68,17 @@ using namespace llvm; namespace { -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ -class AFLCoverage : public PassInfoMixin { - - public: - AFLCoverage() { - -#else class AFLCoverage : public ModulePass { public: static char ID; AFLCoverage() : ModulePass(ID) { -#endif - initInstrumentList(); } -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); -#else bool runOnModule(Module &M) override; -#endif protected: uint32_t ngram_size = 0; @@ -114,55 +92,7 @@ class AFLCoverage : public ModulePass { } // namespace -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ -extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK -llvmGetPassPluginInfo() { - - return {LLVM_PLUGIN_API_VERSION, "AFLCoverage", "v0.1", - /* lambda to insert our pass into the pass pipeline. */ - [](PassBuilder &PB) { - - #if 1 - #if LLVM_VERSION_MAJOR <= 13 - using OptimizationLevel = typename PassBuilder::OptimizationLevel; - #endif - PB.registerOptimizerLastEPCallback( - [](ModulePassManager &MPM, OptimizationLevel OL) { - - MPM.addPass(AFLCoverage()); - - }); - - /* TODO LTO registration */ - #else - using PipelineElement = typename PassBuilder::PipelineElement; - PB.registerPipelineParsingCallback([](StringRef Name, - ModulePassManager &MPM, - ArrayRef) { - - if (Name == "AFLCoverage") { - - MPM.addPass(AFLCoverage()); - return true; - - } else { - - return false; - - } - - }); - - #endif - - }}; - -} - -#else - char AFLCoverage::ID = 0; -#endif /* needed up to 3.9.0 */ #if LLVM_VERSION_MAJOR == 3 && \ @@ -188,15 +118,8 @@ uint64_t PowerOf2Ceil(unsigned in) { (LLVM_VERSION_MAJOR == 4 && LLVM_VERSION_PATCH >= 1) #define AFL_HAVE_VECTOR_INTRINSICS 1 #endif - -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ -PreservedAnalyses AFLCoverage::run(Module &M, ModuleAnalysisManager &MAM) { - -#else bool AFLCoverage::runOnModule(Module &M) { -#endif - LLVMContext &C = M.getContext(); IntegerType *Int8Ty = IntegerType::getInt8Ty(C); @@ -210,10 +133,6 @@ bool AFLCoverage::runOnModule(Module &M) { u32 rand_seed; unsigned int cur_loc = 0; -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - auto PA = PreservedAnalyses::all(); -#endif - /* Setup random() so we get Actually Random(TM) outputs from AFL_R() */ gettimeofday(&tv, &tz); rand_seed = tv.tv_sec ^ tv.tv_usec ^ getpid(); @@ -1051,15 +970,10 @@ bool AFLCoverage::runOnModule(Module &M) { } -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - return PA; -#else return true; -#endif } -#if LLVM_VERSION_MAJOR < 11 /* use old pass manager */ static void registerAFLPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -1072,5 +986,4 @@ static RegisterStandardPasses RegisterAFLPass( static RegisterStandardPasses RegisterAFLPass0( PassManagerBuilder::EP_EnabledOnOptLevel0, registerAFLPass); -#endif diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index f2f0de82..a7b7aac8 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -28,16 +28,11 @@ #include "llvm/Config/llvm-config.h" #include "llvm/ADT/Statistic.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#if LLVM_MAJOR >= 11 -// #include "llvm/Passes/PassPlugin.h" -// #include "llvm/Passes/PassBuilder.h" - #include "llvm/IR/PassManager.h" -#else - #include "llvm/IR/LegacyPassManager.h" -#endif +#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" @@ -59,15 +54,6 @@ using namespace llvm; namespace { -#if LLVM_MAJOR >= 11 /* use new pass manager */ -class CmpLogInstructions : public PassInfoMixin { - public: - CmpLogInstructions() { - - initInstrumentList(); - - } -#else class CmpLogInstructions : public ModulePass { public: @@ -77,11 +63,7 @@ class CmpLogInstructions : public ModulePass { initInstrumentList(); } -#endif -#if LLVM_MAJOR >= 11 /* use new pass manager */ - PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); -#else bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR < 4 @@ -94,7 +76,6 @@ class CmpLogInstructions : public ModulePass { return "cmplog instructions"; } -#endif private: bool hookInstrs(Module &M); @@ -103,9 +84,7 @@ class CmpLogInstructions : public ModulePass { } // namespace -#if LLVM_MAJOR <= 10 /* use old pass manager */ char CmpLogInstructions::ID = 0; -#endif template Iterator Unique(Iterator first, Iterator last) { @@ -588,12 +567,7 @@ bool CmpLogInstructions::hookInstrs(Module &M) { } -#if LLVM_MAJOR >= 11 /* use new pass manager */ -PreservedAnalyses CmpLogInstructions::run(Module & M, - ModuleAnalysisManager &MAM) { -#else bool CmpLogInstructions::runOnModule(Module &M) { -#endif if (getenv("AFL_QUIET") == NULL) printf("Running cmplog-instructions-pass by andreafioraldi@gmail.com\n"); @@ -602,15 +576,10 @@ bool CmpLogInstructions::runOnModule(Module &M) { hookInstrs(M); verifyModule(M); -#if LLVM_MAJOR >= 11 /* use new pass manager */ - return PreservedAnalyses::all(); -#else return true; -#endif } -#if LLVM_MAJOR < 11 /* use old pass manager */ static void registerCmpLogInstructionsPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -630,4 +599,4 @@ static RegisterStandardPasses RegisterCmpLogInstructionsPassLTO( PassManagerBuilder::EP_FullLinkTimeOptimizationLast, registerCmpLogInstructionsPass); #endif -#endif + diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index 2eb3d77b..1ec2bbfe 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -26,23 +26,14 @@ #include "llvm/ADT/Statistic.h" #include "llvm/IR/IRBuilder.h" -#if LLVM_MAJOR >= 11 /* use new pass manager */ - #include "llvm/Passes/PassPlugin.h" - #include "llvm/Passes/PassBuilder.h" - #include "llvm/IR/PassManager.h" -#else - #include "llvm/IR/LegacyPassManager.h" - #include "llvm/Transforms/IPO/PassManagerBuilder.h" -#endif +#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR >= 14 /* how about stable interfaces? */ - #include "llvm/Passes/OptimizationLevel.h" -#endif #if LLVM_VERSION_MAJOR > 3 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) @@ -61,30 +52,28 @@ using namespace llvm; namespace { -#if LLVM_MAJOR >= 11 /* use new pass manager */ -class CompareTransform : public PassInfoMixin { - - public: - CompareTransform() { - -#else class CompareTransform : public ModulePass { public: static char ID; CompareTransform() : ModulePass(ID) { -#endif - initInstrumentList(); } -#if LLVM_MAJOR >= 11 /* use new pass manager */ - PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); -#else bool runOnModule(Module &M) override; + +#if LLVM_VERSION_MAJOR < 4 + const char *getPassName() const override { + +#else + StringRef getPassName() const override { + #endif + return "transforms compare functions"; + + } private: bool transformCmps(Module &M, const bool processStrcmp, @@ -96,54 +85,7 @@ class CompareTransform : public ModulePass { } // namespace -#if LLVM_MAJOR >= 11 /* use new pass manager */ -extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK -llvmGetPassPluginInfo() { - - return {LLVM_PLUGIN_API_VERSION, "comparetransform", "v0.1", - /* lambda to insert our pass into the pass pipeline. */ - [](PassBuilder &PB) { - - #if 1 - #if LLVM_VERSION_MAJOR <= 13 - using OptimizationLevel = typename PassBuilder::OptimizationLevel; - #endif - PB.registerOptimizerLastEPCallback( - [](ModulePassManager &MPM, OptimizationLevel OL) { - - MPM.addPass(CompareTransform()); - - }); - - /* TODO LTO registration */ - #else - using PipelineElement = typename PassBuilder::PipelineElement; - PB.registerPipelineParsingCallback([](StringRef Name, - ModulePassManager &MPM, - ArrayRef) { - - if (Name == "comparetransform") { - - MPM.addPass(CompareTransform()); - return true; - - } else { - - return false; - - } - - }); - - #endif - - }}; - -} - -#else char CompareTransform::ID = 0; -#endif bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, const bool processMemcmp, @@ -442,7 +384,6 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, bool isSizedcmp = false; bool isCaseInsensitive = false; Function * Callee = callInst->getCalledFunction(); - if (Callee) { isMemcmp = Callee->getName().compare("memcmp") == 0; @@ -507,12 +448,13 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, // the following is in general OK, but strncmp is sometimes used in binary // data structures and this can result in crashes :( so it is commented out /* - // add null termination character implicit in c strings - if (!isMemcmp && TmpConstStr[TmpConstStr.length() - 1]) { - TmpConstStr.append("\0", 1); + // add null termination character implicit in c strings + if (!isMemcmp && TmpConstStr[TmpConstStr.length() - 1]) { - } + TmpConstStr.append("\0", 1); + + } */ @@ -656,14 +598,8 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, } -#if LLVM_MAJOR >= 11 /* use new pass manager */ -PreservedAnalyses CompareTransform::run(Module &M, ModuleAnalysisManager &MAM) { - -#else bool CompareTransform::runOnModule(Module &M) { -#endif - if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) printf( "Running compare-transform-pass by laf.intel@gmail.com, extended by " @@ -671,28 +607,13 @@ bool CompareTransform::runOnModule(Module &M) { else be_quiet = 1; -#if LLVM_MAJOR >= 11 /* use new pass manager */ - auto PA = PreservedAnalyses::all(); -#endif - transformCmps(M, true, true, true, true, true); verifyModule(M); -#if LLVM_MAJOR >= 11 /* use new pass manager */ - /* if (modified) { - - PA.abandon(); - - }*/ - - return PA; -#else return true; -#endif } -#if LLVM_MAJOR < 11 /* use old pass manager */ static void registerCompTransPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -707,9 +628,8 @@ static RegisterStandardPasses RegisterCompTransPass( static RegisterStandardPasses RegisterCompTransPass0( PassManagerBuilder::EP_EnabledOnOptLevel0, registerCompTransPass); - #if LLVM_VERSION_MAJOR >= 11 +#if LLVM_VERSION_MAJOR >= 11 static RegisterStandardPasses RegisterCompTransPassLTO( PassManagerBuilder::EP_FullLinkTimeOptimizationLast, registerCompTransPass); - #endif #endif diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index 8637398f..d1254e40 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -1,7 +1,6 @@ /* * Copyright 2016 laf-intel * extended for floating point by Heiko Eißfeldt - * adapted to new pass manager by Heiko Eißfeldt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,20 +28,10 @@ #include "llvm/Pass.h" #include "llvm/Support/raw_ostream.h" - -#if LLVM_MAJOR >= 11 - #include "llvm/Passes/PassPlugin.h" - #include "llvm/Passes/PassBuilder.h" - #include "llvm/IR/PassManager.h" -#else - #include "llvm/IR/LegacyPassManager.h" - #include "llvm/Transforms/IPO/PassManagerBuilder.h" -#endif +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/IR/Module.h" -#if LLVM_VERSION_MAJOR >= 14 /* how about stable interfaces? */ - #include "llvm/Passes/OptimizationLevel.h" -#endif #include "llvm/IR/IRBuilder.h" #if LLVM_VERSION_MAJOR > 3 || \ @@ -64,31 +53,27 @@ using namespace llvm; namespace { -#if LLVM_MAJOR >= 11 -class SplitComparesTransform : public PassInfoMixin { - - public: - // static char ID; - SplitComparesTransform() : enableFPSplit(0) { - -#else class SplitComparesTransform : public ModulePass { public: static char ID; SplitComparesTransform() : ModulePass(ID), enableFPSplit(0) { -#endif - initInstrumentList(); } -#if LLVM_MAJOR >= 11 - PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); -#else bool runOnModule(Module &M) override; +#if LLVM_VERSION_MAJOR >= 4 + StringRef getPassName() const override { + +#else + const char *getPassName() const override { + #endif + return "AFL_SplitComparesTransform"; + + } private: int enableFPSplit; @@ -177,54 +162,7 @@ class SplitComparesTransform : public ModulePass { } // namespace -#if LLVM_MAJOR >= 11 -extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK -llvmGetPassPluginInfo() { - - return {LLVM_PLUGIN_API_VERSION, "splitcompares", "v0.1", - /* lambda to insert our pass into the pass pipeline. */ - [](PassBuilder &PB) { - - #if 1 - #if LLVM_VERSION_MAJOR <= 13 - using OptimizationLevel = typename PassBuilder::OptimizationLevel; - #endif - PB.registerOptimizerLastEPCallback( - [](ModulePassManager &MPM, OptimizationLevel OL) { - - MPM.addPass(SplitComparesTransform()); - - }); - - /* TODO LTO registration */ - #else - using PipelineElement = typename PassBuilder::PipelineElement; - PB.registerPipelineParsingCallback([](StringRef Name, - ModulePassManager &MPM, - ArrayRef) { - - if (Name == "splitcompares") { - - MPM.addPass(SplitComparesTransform()); - return true; - - } else { - - return false; - - } - - }); - - #endif - - }}; - -} - -#else char SplitComparesTransform::ID = 0; -#endif /// This function splits FCMP instructions with xGE or xLE predicates into two /// FCMP instructions with predicate xGT or xLT and EQ @@ -1378,15 +1316,8 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { } -#if LLVM_MAJOR >= 11 -PreservedAnalyses SplitComparesTransform::run(Module & M, - ModuleAnalysisManager &MAM) { - -#else bool SplitComparesTransform::runOnModule(Module &M) { -#endif - char *bitw_env = getenv("AFL_LLVM_LAF_SPLIT_COMPARES_BITW"); if (!bitw_env) bitw_env = getenv("LAF_SPLIT_COMPARES_BITW"); if (bitw_env) { target_bitwidth = atoi(bitw_env); } @@ -1396,7 +1327,7 @@ bool SplitComparesTransform::runOnModule(Module &M) { if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) { - errs() << "Split-compare-newpass by laf.intel@gmail.com, extended by " + errs() << "Split-compare-pass by laf.intel@gmail.com, extended by " "heiko@hexco.de (splitting icmp to " << target_bitwidth << " bit)\n"; @@ -1408,10 +1339,6 @@ bool SplitComparesTransform::runOnModule(Module &M) { } -#if LLVM_MAJOR >= 11 - auto PA = PreservedAnalyses::all(); -#endif - if (enableFPSplit) { count = splitFPCompares(M); @@ -1444,16 +1371,7 @@ bool SplitComparesTransform::runOnModule(Module &M) { auto op0 = CI->getOperand(0); auto op1 = CI->getOperand(1); - if (!op0 || !op1) { - -#if LLVM_MAJOR >= 11 - return PA; -#else - return false; -#endif - - } - + if (!op0 || !op1) { return false; } auto iTy1 = dyn_cast(op0->getType()); if (iTy1 && isa(op1->getType())) { @@ -1502,29 +1420,10 @@ bool SplitComparesTransform::runOnModule(Module &M) { } - if ((isatty(2) && getenv("AFL_QUIET") == NULL) || - getenv("AFL_DEBUG") != NULL) { - - errs() << count << " comparisons found\n"; - - } - -#if LLVM_MAJOR >= 11 - /* if (modified) { - - PA.abandon(); - - }*/ - - return PA; -#else return true; -#endif } -#if LLVM_MAJOR < 11 /* use old pass manager */ - static void registerSplitComparesPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -1538,15 +1437,14 @@ static RegisterStandardPasses RegisterSplitComparesPass( static RegisterStandardPasses RegisterSplitComparesTransPass0( PassManagerBuilder::EP_EnabledOnOptLevel0, registerSplitComparesPass); - #if LLVM_VERSION_MAJOR >= 11 +#if LLVM_VERSION_MAJOR >= 11 static RegisterStandardPasses RegisterSplitComparesTransPassLTO( PassManagerBuilder::EP_FullLinkTimeOptimizationLast, registerSplitComparesPass); - #endif +#endif static RegisterPass X("splitcompares", "AFL++ split compares", true /* Only looks at CFG */, true /* Analysis Pass */); -#endif diff --git a/instrumentation/split-switches-pass.so.cc b/instrumentation/split-switches-pass.so.cc index e0a96ac9..1e32a31d 100644 --- a/instrumentation/split-switches-pass.so.cc +++ b/instrumentation/split-switches-pass.so.cc @@ -27,23 +27,14 @@ #include "llvm/ADT/Statistic.h" #include "llvm/IR/IRBuilder.h" -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - #include "llvm/Passes/PassPlugin.h" - #include "llvm/Passes/PassBuilder.h" - #include "llvm/IR/PassManager.h" -#else - #include "llvm/IR/LegacyPassManager.h" - #include "llvm/Transforms/IPO/PassManagerBuilder.h" -#endif +#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR >= 14 /* how about stable interfaces? */ - #include "llvm/Passes/OptimizationLevel.h" -#endif #include "llvm/IR/IRBuilder.h" #if LLVM_VERSION_MAJOR > 3 || \ @@ -63,42 +54,29 @@ using namespace llvm; namespace { -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ -class SplitSwitchesTransform : public PassInfoMixin { - - public: - SplitSwitchesTransform() { - -#else class SplitSwitchesTransform : public ModulePass { public: static char ID; SplitSwitchesTransform() : ModulePass(ID) { -#endif initInstrumentList(); } -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); -#else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; - #if LLVM_VERSION_MAJOR >= 4 +#if LLVM_VERSION_MAJOR >= 4 StringRef getPassName() const override { - #else +#else const char *getPassName() const override { - #endif +#endif return "splits switch constructs"; } -#endif - struct CaseExpr { ConstantInt *Val; @@ -125,54 +103,7 @@ class SplitSwitchesTransform : public ModulePass { } // namespace -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ -extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK -llvmGetPassPluginInfo() { - - return {LLVM_PLUGIN_API_VERSION, "splitswitches", "v0.1", - /* lambda to insert our pass into the pass pipeline. */ - [](PassBuilder &PB) { - - #if 1 - #if LLVM_VERSION_MAJOR <= 13 - using OptimizationLevel = typename PassBuilder::OptimizationLevel; - #endif - PB.registerOptimizerLastEPCallback( - [](ModulePassManager &MPM, OptimizationLevel OL) { - - MPM.addPass(SplitSwitchesTransform()); - - }); - - /* TODO LTO registration */ - #else - using PipelineElement = typename PassBuilder::PipelineElement; - PB.registerPipelineParsingCallback([](StringRef Name, - ModulePassManager &MPM, - ArrayRef) { - - if (Name == "splitswitches") { - - MPM.addPass(SplitSwitchesTransform()); - return true; - - } else { - - return false; - - } - - }); - - #endif - - }}; - -} - -#else char SplitSwitchesTransform::ID = 0; -#endif /* switchConvert - Transform simple list of Cases into list of CaseRange's */ BasicBlock *SplitSwitchesTransform::switchConvert( @@ -484,42 +415,19 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { } -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ -PreservedAnalyses SplitSwitchesTransform::run(Module & M, - ModuleAnalysisManager &MAM) { - -#else bool SplitSwitchesTransform::runOnModule(Module &M) { -#endif - if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) printf("Running split-switches-pass by laf.intel@gmail.com\n"); else be_quiet = 1; - -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - auto PA = PreservedAnalyses::all(); -#endif - splitSwitches(M); verifyModule(M); -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - /* if (modified) { - - PA.abandon(); - - }*/ - - return PA; -#else return true; -#endif } -#if LLVM_VERSION_MAJOR < 11 /* use old pass manager */ static void registerSplitSwitchesTransPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -534,10 +442,9 @@ static RegisterStandardPasses RegisterSplitSwitchesTransPass( static RegisterStandardPasses RegisterSplitSwitchesTransPass0( PassManagerBuilder::EP_EnabledOnOptLevel0, registerSplitSwitchesTransPass); - #if LLVM_VERSION_MAJOR >= 11 +#if LLVM_VERSION_MAJOR >= 11 static RegisterStandardPasses RegisterSplitSwitchesTransPassLTO( PassManagerBuilder::EP_FullLinkTimeOptimizationLast, registerSplitSwitchesTransPass); - #endif #endif diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl index 8809a2b2..002e4739 160000 --- a/qemu_mode/qemuafl +++ b/qemu_mode/qemuafl @@ -1 +1 @@ -Subproject commit 8809a2b2ebf089d3427dd8f6a0044bcc2e13b389 +Subproject commit 002e473939a350854d56f67ce7b2e2d9706b8bca diff --git a/src/afl-cc.c b/src/afl-cc.c index c70f193c..9c6e9b3e 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -462,17 +462,11 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_MAJOR >= 11 /* use new pass manager */ - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); -#else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/split-switches-pass.so", obj_path); -#endif } @@ -488,17 +482,11 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_MAJOR >= 11 /* use new pass manager */ - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - cc_params[cc_par_cnt++] = alloc_printf( - "-fpass-plugin=%s/compare-transform-pass.so", obj_path); -#else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/compare-transform-pass.so", obj_path); -#endif } @@ -514,18 +502,11 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_MAJOR >= 11 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/split-compares-pass.so", obj_path); -// cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; -#else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/split-compares-pass.so", obj_path); -#endif } @@ -555,17 +536,11 @@ static void edit_params(u32 argc, char **argv, char **envp) { alloc_printf("%s/cmplog-switches-pass.so", obj_path); // reuse split switches from laf -#if LLVM_MAJOR >= 11 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/split-switches-pass.so", obj_path); -#else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/split-switches-pass.so", obj_path); -#endif } @@ -573,6 +548,10 @@ static void edit_params(u32 argc, char **argv, char **envp) { } +#if LLVM_MAJOR >= 13 + // fuck you llvm 13 + cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; +#endif if (lto_mode && !have_c) { @@ -612,7 +591,6 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { - cc_params[cc_par_cnt++] = "-fno-experimental-new-pass-manager"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; @@ -645,16 +623,10 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_MAJOR >= 11 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/afl-llvm-pass.so", obj_path); -#else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); -#endif } @@ -671,17 +643,11 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { -#if LLVM_MAJOR >= 11 - cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; - cc_params[cc_par_cnt++] = - alloc_printf("-fpass-plugin=%s/cmplog-instructions-pass.so", obj_path); -#else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/cmplog-instructions-pass.so", obj_path); -#endif cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; -- cgit 1.4.1 From 655b63d2b289035c951dad744f1aa68aa61d0f50 Mon Sep 17 00:00:00 2001 From: eternalsakura Date: Sun, 28 Nov 2021 22:28:38 +0800 Subject: [fix] Custom mutator does not implement all three trim APIs, standard trimming will be used, but now the actual implementation does not match the description, fix this problem --- src/afl-fuzz-mutators.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 5332b9fe..c1d3a683 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -255,6 +255,7 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { mutator->afl_custom_init_trim = dlsym(dh, "afl_custom_init_trim"); if (!mutator->afl_custom_init_trim) { + notrim = 1; ACTF("optional symbol 'afl_custom_init_trim' not found."); } @@ -263,6 +264,7 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { mutator->afl_custom_trim = dlsym(dh, "afl_custom_trim"); if (!mutator->afl_custom_trim) { + notrim = 1; ACTF("optional symbol 'afl_custom_trim' not found."); } @@ -271,6 +273,7 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { mutator->afl_custom_post_trim = dlsym(dh, "afl_custom_post_trim"); if (!mutator->afl_custom_post_trim) { + notrim = 1; ACTF("optional symbol 'afl_custom_post_trim' not found."); } -- cgit 1.4.1 From 22827e807035a50394bac6a37406b19109fc655f Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 1 Dec 2021 11:38:55 +0100 Subject: unify LLVM_VERSION_... usage --- instrumentation/SanitizerCoverageLTO.so.cc | 10 +-- instrumentation/SanitizerCoveragePCGUARD.so.cc | 24 ++--- instrumentation/afl-llvm-common.cc | 2 +- instrumentation/afl-llvm-lto-instrumentlist.so.cc | 8 +- instrumentation/afl-llvm-pass.so.cc | 30 ++----- instrumentation/cmplog-instructions-pass.cc | 82 ++++++++--------- instrumentation/cmplog-routines-pass.cc | 104 +++++++++++----------- instrumentation/cmplog-switches-pass.cc | 50 +++++------ instrumentation/compare-transform-pass.so.cc | 32 +++---- instrumentation/split-compares-pass.so.cc | 6 +- instrumentation/split-switches-pass.so.cc | 8 +- src/afl-cc.c | 24 ++--- 12 files changed, 183 insertions(+), 197 deletions(-) (limited to 'src') diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index 37726607..bff85a0a 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -1325,7 +1325,7 @@ void ModuleSanitizerCoverage::instrumentFunction( } else -#if LLVM_VERSION_MAJOR > 13 +#if LLVM_VERSION_MAJOR >= 14 if (t->getTypeID() == llvm::Type::FixedVectorTyID) { FixedVectorType *tt = dyn_cast(t); @@ -1468,7 +1468,7 @@ GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection( *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, Constant::getNullValue(ArrayTy), "__sancov_gen_"); -#if LLVM_VERSION_MAJOR > 12 +#if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && (TargetTriple.isOSBinFormatELF() || !F.isInterposable())) if (auto Comdat = getOrCreateFunctionComdat(F, TargetTriple)) @@ -1628,10 +1628,10 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, if (Options.TracePC) { IRB.CreateCall(SanCovTracePC) -#if LLVM_VERSION_MAJOR < 12 - ->cannotMerge(); // gets the PC using GET_CALLER_PC. -#else +#if LLVM_VERSION_MAJOR >= 12 ->setCannotMerge(); // gets the PC using GET_CALLER_PC. +#else + ->cannotMerge(); // gets the PC using GET_CALLER_PC. #endif } diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 76bb2448..3574b0e4 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -36,7 +36,8 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/SpecialCaseList.h" -#if LLVM_MAJOR > 10 || (LLVM_MAJOR == 10 && LLVM_MINOR > 0) +#if LLVM_VERSION_MAJOR >= 11 || \ + (LLVM_VERSION_MAJOR == 10 && LLVM_VERSION_MINOR >= 1) #include "llvm/Support/VirtualFileSystem.h" #endif #include "llvm/Support/raw_ostream.h" @@ -127,7 +128,7 @@ class ModuleSanitizerCoverage { public: ModuleSanitizerCoverage( const SanitizerCoverageOptions &Options = SanitizerCoverageOptions() -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , const SpecialCaseList *Allowlist = nullptr, const SpecialCaseList *Blocklist = nullptr @@ -215,7 +216,7 @@ class ModuleSanitizerCoverageLegacyPass : public ModulePass { public: ModuleSanitizerCoverageLegacyPass( const SanitizerCoverageOptions &Options = SanitizerCoverageOptions() -#if LLVM_VERSION_MAJOR > 10 +#if LLVM_VERSION_MAJOR >= 11 , const std::vector &AllowlistFiles = std::vector(), @@ -233,7 +234,7 @@ class ModuleSanitizerCoverageLegacyPass : public ModulePass { bool runOnModule(Module &M) override { ModuleSanitizerCoverage ModuleSancov(Options -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , Allowlist.get(), Blocklist.get() #endif @@ -283,7 +284,7 @@ PreservedAnalyses ModuleSanitizerCoveragePass::run(Module & M, ModuleAnalysisManager &MAM) { ModuleSanitizerCoverage ModuleSancov(Options -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , Allowlist.get(), Blocklist.get() #endif @@ -749,7 +750,7 @@ GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection( *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, Constant::getNullValue(ArrayTy), "__sancov_gen_"); -#if LLVM_VERSION_MAJOR > 12 +#if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && (TargetTriple.isOSBinFormatELF() || !F.isInterposable())) if (auto Comdat = getOrCreateFunctionComdat(F, TargetTriple)) @@ -762,7 +763,8 @@ GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection( #endif Array->setSection(getSectionName(Section)); -#if LLVM_MAJOR > 10 || (LLVM_MAJOR == 10 && LLVM_MINOR > 0) +#if (LLVM_VERSION_MAJOR >= 11) || \ + (LLVM_VERSION_MAJOR == 10 && LLVM_VERSION_MINOR >= 1) Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedSize())); #else Array->setAlignment(Align(4)); // cheating @@ -868,7 +870,7 @@ bool ModuleSanitizerCoverage::InjectCoverage(Function & F, } -#if LLVM__MAJOR > 11 +#if (LLVM_VERSION_MAJOR >= 12) else if (t->getTypeID() == llvm::Type::FixedVectorTyID) { FixedVectorType *tt = dyn_cast(t); @@ -962,7 +964,7 @@ bool ModuleSanitizerCoverage::InjectCoverage(Function & F, } else -#if LLVM_VERSION_MAJOR > 13 +#if LLVM_VERSION_MAJOR >= 14 if (t->getTypeID() == llvm::Type::FixedVectorTyID) { FixedVectorType *tt = dyn_cast(t); @@ -1484,7 +1486,7 @@ INITIALIZE_PASS_END(ModuleSanitizerCoverageLegacyPass, "sancov", ModulePass *llvm::createModuleSanitizerCoverageLegacyPassPass( const SanitizerCoverageOptions &Options -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , const std::vector &AllowlistFiles, const std::vector &BlocklistFiles @@ -1492,7 +1494,7 @@ ModulePass *llvm::createModuleSanitizerCoverageLegacyPassPass( ) { return new ModuleSanitizerCoverageLegacyPass(Options -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , AllowlistFiles, BlocklistFiles #endif diff --git a/instrumentation/afl-llvm-common.cc b/instrumentation/afl-llvm-common.cc index 3239ea91..e5e367a7 100644 --- a/instrumentation/afl-llvm-common.cc +++ b/instrumentation/afl-llvm-common.cc @@ -281,7 +281,7 @@ void scanForDangerousFunctions(llvm::Module *M) { if (!M) return; -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 9) for (GlobalIFunc &IF : M->ifuncs()) { diff --git a/instrumentation/afl-llvm-lto-instrumentlist.so.cc b/instrumentation/afl-llvm-lto-instrumentlist.so.cc index cf26f912..906af879 100644 --- a/instrumentation/afl-llvm-lto-instrumentlist.so.cc +++ b/instrumentation/afl-llvm-lto-instrumentlist.so.cc @@ -116,14 +116,14 @@ bool AFLcheckIfInstrument::runOnModule(Module &M) { auto & Ctx = F.getContext(); AttributeList Attrs = F.getAttributes(); -#if LLVM_VERSION_MAJOR < 14 +#if LLVM_VERSION_MAJOR >= 14 + AttributeList NewAttrs = Attrs.addFnAttribute(Ctx, "skipinstrument"); + F.setAttributes(NewAttrs); +#else AttrBuilder NewAttrs; NewAttrs.addAttribute("skipinstrument"); F.setAttributes( Attrs.addAttributes(Ctx, AttributeList::FunctionIndex, NewAttrs)); -#else - AttributeList NewAttrs = Attrs.addFnAttribute(Ctx, "skipinstrument"); - F.setAttributes(NewAttrs); #endif } diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index 21ce0cf9..8e22fde8 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -52,7 +52,7 @@ typedef long double max_align_t; #include "llvm/Support/MathExtras.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/DebugInfo.h" #include "llvm/IR/CFG.h" @@ -114,7 +114,7 @@ uint64_t PowerOf2Ceil(unsigned in) { #endif /* #if LLVM_VERSION_STRING >= "4.0.1" */ -#if LLVM_VERSION_MAJOR > 4 || \ +#if LLVM_VERSION_MAJOR >= 5 || \ (LLVM_VERSION_MAJOR == 4 && LLVM_VERSION_PATCH >= 1) #define AFL_HAVE_VECTOR_INTRINSICS 1 #endif @@ -662,22 +662,7 @@ bool AFLCoverage::runOnModule(Module &M) { /* Update bitmap */ if (use_threadsafe_counters) { /* Atomic */ - /* - #if LLVM_VERSION_MAJOR < 9 - if (neverZero_counters_str != - NULL) { // with llvm 9 we make this the default as the bug - in llvm - // is then fixed - #else - if (!skip_nozero) { - - #endif - // register MapPtrIdx in a todo list - todo.push_back(MapPtrIdx); - - } else { - - */ + IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One, #if LLVM_VERSION_MAJOR >= 13 llvm::MaybeAlign(1), @@ -696,13 +681,12 @@ bool AFLCoverage::runOnModule(Module &M) { Value *Incr = IRB.CreateAdd(Counter, One); -#if LLVM_VERSION_MAJOR < 9 - if (neverZero_counters_str != - NULL) { // with llvm 9 we make this the default as the bug in llvm - // is then fixed -#else +#if LLVM_VERSION_MAJOR >= 9 if (!skip_nozero) { +#else + if (neverZero_counters_str != NULL) { + #endif /* hexcoder: Realize a counter that skips zero during overflow. * Once this counter reaches its maximum value, it next increments to diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index a7b7aac8..07f80b2c 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -37,7 +37,7 @@ #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -66,11 +66,11 @@ class CmpLogInstructions : public ModulePass { bool runOnModule(Module &M) override; -#if LLVM_VERSION_MAJOR < 4 - const char *getPassName() const override { +#if LLVM_VERSION_MAJOR >= 4 + StringRef getPassName() const override { #else - StringRef getPassName() const override { + const char *getPassName() const override { #endif return "cmplog instructions"; @@ -113,10 +113,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { IntegerType *Int64Ty = IntegerType::getInt64Ty(C); IntegerType *Int128Ty = IntegerType::getInt128Ty(C); -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c1 = M.getOrInsertFunction("__cmplog_ins_hook1", VoidTy, Int8Ty, Int8Ty, Int8Ty @@ -125,16 +125,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns1 = cast(c1); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns1 = c1; +#else + Function *cmplogHookIns1 = cast(c1); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c2 = M.getOrInsertFunction("__cmplog_ins_hook2", VoidTy, Int16Ty, Int16Ty, Int8Ty @@ -143,16 +143,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns2 = cast(c2); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns2 = c2; +#else + Function *cmplogHookIns2 = cast(c2); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c4 = M.getOrInsertFunction("__cmplog_ins_hook4", VoidTy, Int32Ty, Int32Ty, Int8Ty @@ -161,16 +161,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns4 = cast(c4); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns4 = c4; +#else + Function *cmplogHookIns4 = cast(c4); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c8 = M.getOrInsertFunction("__cmplog_ins_hook8", VoidTy, Int64Ty, Int64Ty, Int8Ty @@ -179,16 +179,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns8 = cast(c8); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns8 = c8; +#else + Function *cmplogHookIns8 = cast(c8); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c16 = M.getOrInsertFunction("__cmplog_ins_hook16", VoidTy, Int128Ty, Int128Ty, Int8Ty @@ -203,10 +203,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { FunctionCallee cmplogHookIns16 = c16; #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif cN = M.getOrInsertFunction("__cmplog_ins_hookN", VoidTy, Int128Ty, Int128Ty, Int8Ty, Int8Ty @@ -215,10 +215,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookInsN = cast(cN); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookInsN = cN; +#else + Function *cmplogHookInsN = cast(cN); #endif GlobalVariable *AFLCmplogPtr = M.getNamedGlobal("__afl_cmp_map"); @@ -338,7 +338,7 @@ bool CmpLogInstructions::hookInstrs(Module &M) { } -#if LLVM_MAJOR > 11 +#if (LLVM_VERSION_MAJOR >= 12) vector_cnt = tt->getElementCount().getKnownMinValue(); ty0 = tt->getElementType(); #endif @@ -359,7 +359,7 @@ bool CmpLogInstructions::hookInstrs(Module &M) { max_size = 80; else if (ty0->isFP128Ty() || ty0->isPPC_FP128Ty()) max_size = 128; -#if LLVM_MAJOR > 11 +#if (LLVM_VERSION_MAJOR >= 12) else if (ty0->getTypeID() != llvm::Type::PointerTyID && !be_quiet) fprintf(stderr, "Warning: unsupported cmp type for cmplog: %u!\n", ty0->getTypeID()); @@ -371,7 +371,7 @@ bool CmpLogInstructions::hookInstrs(Module &M) { if (ty0->isVectorTy()) { -#if LLVM_MAJOR > 11 +#if (LLVM_VERSION_MAJOR >= 12) VectorType *tt = dyn_cast(ty0); if (!tt) { @@ -397,7 +397,7 @@ bool CmpLogInstructions::hookInstrs(Module &M) { } else { -#if LLVM_MAJOR > 11 +#if (LLVM_VERSION_MAJOR >= 12) if (ty0->getTypeID() != llvm::Type::PointerTyID && !be_quiet) { fprintf(stderr, "Warning: unsupported cmp type for cmplog: %u\n", diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index fb514edc..0565875e 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -36,7 +36,7 @@ #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -65,11 +65,11 @@ class CmpLogRoutines : public ModulePass { bool runOnModule(Module &M) override; -#if LLVM_VERSION_MAJOR < 4 - const char *getPassName() const override { +#if LLVM_VERSION_MAJOR >= 4 + StringRef getPassName() const override { #else - StringRef getPassName() const override { + const char *getPassName() const override { #endif return "cmplog routines"; @@ -97,10 +97,10 @@ bool CmpLogRoutines::hookRtns(Module &M) { IntegerType *Int64Ty = IntegerType::getInt64Ty(C); PointerType *i8PtrTy = PointerType::get(Int8Ty, 0); -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c = M.getOrInsertFunction("__cmplog_rtn_hook", VoidTy, i8PtrTy, i8PtrTy #if LLVM_VERSION_MAJOR < 5 @@ -108,16 +108,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookFn = cast(c); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookFn = c; +#else + Function *cmplogHookFn = cast(c); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c1 = M.getOrInsertFunction("__cmplog_rtn_llvm_stdstring_stdstring", VoidTy, i8PtrTy, i8PtrTy @@ -126,16 +126,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogLlvmStdStd = cast(c1); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogLlvmStdStd = c1; +#else + Function *cmplogLlvmStdStd = cast(c1); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c2 = M.getOrInsertFunction("__cmplog_rtn_llvm_stdstring_cstring", VoidTy, i8PtrTy, i8PtrTy @@ -144,16 +144,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogLlvmStdC = cast(c2); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogLlvmStdC = c2; +#else + Function *cmplogLlvmStdC = cast(c2); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c3 = M.getOrInsertFunction("__cmplog_rtn_gcc_stdstring_stdstring", VoidTy, i8PtrTy, i8PtrTy @@ -162,16 +162,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogGccStdStd = cast(c3); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogGccStdStd = c3; +#else + Function *cmplogGccStdStd = cast(c3); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c4 = M.getOrInsertFunction("__cmplog_rtn_gcc_stdstring_cstring", VoidTy, i8PtrTy, i8PtrTy @@ -180,16 +180,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogGccStdC = cast(c4); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogGccStdC = c4; +#else + Function *cmplogGccStdC = cast(c4); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c5 = M.getOrInsertFunction("__cmplog_rtn_hook_n", VoidTy, i8PtrTy, i8PtrTy, Int64Ty @@ -198,16 +198,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookFnN = cast(c5); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookFnN = c5; +#else + Function *cmplogHookFnN = cast(c5); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c6 = M.getOrInsertFunction("__cmplog_rtn_hook_strn", VoidTy, i8PtrTy, i8PtrTy, Int64Ty @@ -216,16 +216,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookFnStrN = cast(c6); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookFnStrN = c6; +#else + Function *cmplogHookFnStrN = cast(c6); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c7 = M.getOrInsertFunction("__cmplog_rtn_hook_str", VoidTy, i8PtrTy, i8PtrTy @@ -234,10 +234,10 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookFnStr = cast(c7); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookFnStr = c7; +#else + Function *cmplogHookFnStr = cast(c7); #endif GlobalVariable *AFLCmplogPtr = M.getNamedGlobal("__afl_cmp_map"); diff --git a/instrumentation/cmplog-switches-pass.cc b/instrumentation/cmplog-switches-pass.cc index aa719013..bcd5f8bd 100644 --- a/instrumentation/cmplog-switches-pass.cc +++ b/instrumentation/cmplog-switches-pass.cc @@ -37,7 +37,7 @@ #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -112,10 +112,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { IntegerType *Int32Ty = IntegerType::getInt32Ty(C); IntegerType *Int64Ty = IntegerType::getInt64Ty(C); -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c1 = M.getOrInsertFunction("__cmplog_ins_hook1", VoidTy, Int8Ty, Int8Ty, Int8Ty @@ -124,16 +124,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns1 = cast(c1); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns1 = c1; +#else + Function *cmplogHookIns1 = cast(c1); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c2 = M.getOrInsertFunction("__cmplog_ins_hook2", VoidTy, Int16Ty, Int16Ty, Int8Ty @@ -142,16 +142,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns2 = cast(c2); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns2 = c2; +#else + Function *cmplogHookIns2 = cast(c2); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c4 = M.getOrInsertFunction("__cmplog_ins_hook4", VoidTy, Int32Ty, Int32Ty, Int8Ty @@ -160,16 +160,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns4 = cast(c4); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns4 = c4; +#else + Function *cmplogHookIns4 = cast(c4); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c8 = M.getOrInsertFunction("__cmplog_ins_hook8", VoidTy, Int64Ty, Int64Ty, Int8Ty @@ -178,10 +178,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns8 = cast(c8); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns8 = c8; +#else + Function *cmplogHookIns8 = cast(c8); #endif GlobalVariable *AFLCmplogPtr = M.getNamedGlobal("__afl_cmp_map"); diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index 1ec2bbfe..ef3bd66b 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -35,7 +35,7 @@ #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -64,11 +64,11 @@ class CompareTransform : public ModulePass { bool runOnModule(Module &M) override; -#if LLVM_VERSION_MAJOR < 4 - const char *getPassName() const override { +#if LLVM_VERSION_MAJOR >= 4 + StringRef getPassName() const override { #else - StringRef getPassName() const override { + const char *getPassName() const override { #endif return "transforms compare functions"; @@ -100,17 +100,17 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, IntegerType * Int32Ty = IntegerType::getInt32Ty(C); IntegerType * Int64Ty = IntegerType::getInt64Ty(C); -#if LLVM_VERSION_MAJOR < 9 - Function *tolowerFn; -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee tolowerFn; +#else + Function * tolowerFn; #endif { -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c = M.getOrInsertFunction("tolower", Int32Ty, Int32Ty #if LLVM_VERSION_MAJOR < 5 @@ -118,10 +118,10 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - tolowerFn = cast(c); -#else +#if LLVM_VERSION_MAJOR >= 9 tolowerFn = c; +#else + tolowerFn = cast(c); #endif } @@ -496,10 +496,10 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, PHINode *PN = PHINode::Create( Int32Ty, (next_lenchk_bb ? 2 : 1) * unrollLen + 1, "cmp_phi"); -#if LLVM_VERSION_MAJOR < 8 - TerminatorInst *term = bb->getTerminator(); -#else +#if LLVM_VERSION_MAJOR >= 8 Instruction *term = bb->getTerminator(); +#else + TerminatorInst *term = bb->getTerminator(); #endif BranchInst::Create(next_lenchk_bb ? next_lenchk_bb : next_cmp_bb, bb); term->eraseFromParent(); diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index d1254e40..95485be9 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -34,7 +34,7 @@ #include "llvm/IR/Module.h" #include "llvm/IR/IRBuilder.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -796,7 +796,7 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { LLVMContext &C = M.getContext(); -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 7) const DataLayout &dl = M.getDataLayout(); @@ -1398,7 +1398,7 @@ bool SplitComparesTransform::runOnModule(Module &M) { bool brokenDebug = false; if (verifyModule(M, &errs() -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 9) , &brokenDebug // 9th May 2016 diff --git a/instrumentation/split-switches-pass.so.cc b/instrumentation/split-switches-pass.so.cc index 1e32a31d..c0fa7c9c 100644 --- a/instrumentation/split-switches-pass.so.cc +++ b/instrumentation/split-switches-pass.so.cc @@ -37,7 +37,7 @@ #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/IRBuilder.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -369,10 +369,10 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { CaseVector Cases; for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end(); i != e; ++i) -#if LLVM_VERSION_MAJOR < 5 - Cases.push_back(CaseExpr(i.getCaseValue(), i.getCaseSuccessor())); -#else +#if LLVM_VERSION_MAJOR >= 5 Cases.push_back(CaseExpr(i->getCaseValue(), i->getCaseSuccessor())); +#else + Cases.push_back(CaseExpr(i.getCaseValue(), i.getCaseSuccessor())); #endif /* bugfix thanks to pbst * round up bytesChecked (in case getBitWidth() % 8 != 0) */ diff --git a/src/afl-cc.c b/src/afl-cc.c index 9c6e9b3e..6bdb9572 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -574,7 +574,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { if (instrument_mode == INSTRUMENT_PCGUARD) { -#if LLVM_MAJOR > 10 || (LLVM_MAJOR == 10 && LLVM_MINOR > 0) +#if LLVM_MAJOR >= 11 || (LLVM_MAJOR == 10 && LLVM_MINOR >= 1) #if defined __ANDROID__ || ANDROID cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; instrument_mode = INSTRUMENT_LLVMNATIVE; @@ -1167,7 +1167,7 @@ int main(int argc, char **argv, char **envp) { } -#if (LLVM_MAJOR > 2) +#if (LLVM_MAJOR >= 3) if ((ptr = find_object("SanitizerCoverageLTO.so", argv[0])) != NULL) { @@ -1196,7 +1196,7 @@ int main(int argc, char **argv, char **envp) { } -#if (LLVM_MAJOR > 2) +#if (LLVM_MAJOR >= 3) if (strncmp(callname, "afl-clang-fast", 14) == 0) { @@ -1724,8 +1724,8 @@ int main(int argc, char **argv, char **envp) { compiler_mode == LTO ? " [SELECTED]" : "", have_llvm ? "AVAILABLE" : "unavailable!", compiler_mode == LLVM ? " [SELECTED]" : "", - LLVM_MAJOR > 6 ? "DEFAULT" : " ", - LLVM_MAJOR > 6 ? " " : "DEFAULT", + LLVM_MAJOR >= 7 ? "DEFAULT" : " ", + LLVM_MAJOR >= 7 ? " " : "DEFAULT", have_gcc_plugin ? "AVAILABLE" : "unavailable!", compiler_mode == GCC_PLUGIN ? " [SELECTED]" : "", have_gcc ? "AVAILABLE" : "unavailable!", @@ -1826,12 +1826,12 @@ int main(int argc, char **argv, char **envp) { " AFL_GCC_INSTRUMENT_FILE: enable selective instrumentation by " "filename\n"); -#if LLVM_MAJOR < 9 +#if LLVM_MAJOR >= 9 #define COUNTER_BEHAVIOUR \ - " AFL_LLVM_NOT_ZERO: use cycling trace counters that skip zero\n" + " AFL_LLVM_SKIP_NEVERZERO: do not skip zero on trace counters\n" #else #define COUNTER_BEHAVIOUR \ - " AFL_LLVM_SKIP_NEVERZERO: do not skip zero on trace counters\n" + " AFL_LLVM_NOT_ZERO: use cycling trace counters that skip zero\n" #endif if (have_llvm) SAYF( @@ -1905,7 +1905,7 @@ int main(int argc, char **argv, char **envp) { "consult the README.md, especially section 3.1 about instrumenting " "targets.\n\n"); -#if (LLVM_MAJOR > 2) +#if (LLVM_MAJOR >= 3) if (have_lto) SAYF("afl-cc LTO with ld=%s %s\n", AFL_REAL_LD, AFL_CLANG_FLTO); if (have_llvm) @@ -1967,9 +1967,7 @@ int main(int argc, char **argv, char **envp) { if (instrument_mode == 0 && compiler_mode < GCC_PLUGIN) { -#if LLVM_MAJOR <= 6 - instrument_mode = INSTRUMENT_AFL; -#else +#if LLVM_MAJOR >= 7 #if LLVM_MAJOR < 11 && (LLVM_MAJOR < 10 || LLVM_MINOR < 1) if (have_instr_env) { @@ -1984,6 +1982,8 @@ int main(int argc, char **argv, char **envp) { #endif instrument_mode = INSTRUMENT_PCGUARD; +#else + instrument_mode = INSTRUMENT_AFL; #endif } -- cgit 1.4.1 From 78b0e108a6b57739ff97927f15126c784bb03773 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 1 Dec 2021 12:01:00 +0100 Subject: remove quickly --- src/afl-forkserver.c | 6 ++---- src/afl-fuzz-init.c | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 6320a26b..b871ea8c 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -941,8 +941,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" - " Tip: you can use https://jwilk.net/software/recidivm to " - "quickly\n" + " Tip: you can use https://jwilk.net/software/recidivm to\n" " estimate the required amount of virtual memory for the " "binary.\n\n" @@ -1041,8 +1040,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" - " Tip: you can use https://jwilk.net/software/recidivm to " - "quickly\n" + " Tip: you can use https://jwilk.net/software/recidivm to\n" " estimate the required amount of virtual memory for the " "binary.\n\n" diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index e5a4d3d1..4c030c0a 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -974,8 +974,7 @@ void perform_dry_run(afl_state_t *afl) { MSG_ULIMIT_USAGE " /path/to/binary [...] Date: Thu, 2 Dec 2021 10:41:24 +0100 Subject: fixes --- instrumentation/afl-compiler-rt.o.c | 2 ++ src/afl-fuzz-bitmap.c | 16 ++++++++-------- src/afl-fuzz-stats.c | 6 ++++-- src/afl-showmap.c | 6 +++--- 4 files changed, 17 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 5d198ada..cc73e5ec 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -275,6 +275,8 @@ static void __afl_map_shm(void) { char *id_str = getenv(SHM_ENV_VAR); + if (__afl_final_loc) { ++__afl_final_loc; } // as we count starting 0 + if (__afl_final_loc) { __afl_map_size = __afl_final_loc; diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index a204e374..fa413dcf 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -58,7 +58,7 @@ void write_bitmap(afl_state_t *afl) { u32 count_bits(afl_state_t *afl, u8 *mem) { u32 *ptr = (u32 *)mem; - u32 i = (afl->fsrv.map_size >> 2); + u32 i = ((afl->fsrv.real_map_size + 3) >> 2); u32 ret = 0; while (i--) { @@ -68,7 +68,7 @@ u32 count_bits(afl_state_t *afl, u8 *mem) { /* This gets called on the inverse, virgin bitmap; optimize for sparse data. */ - if (v == 0xffffffff) { + if (likely(v == 0xffffffff)) { ret += 32; continue; @@ -92,14 +92,14 @@ u32 count_bits(afl_state_t *afl, u8 *mem) { u32 count_bytes(afl_state_t *afl, u8 *mem) { u32 *ptr = (u32 *)mem; - u32 i = (afl->fsrv.map_size >> 2); + u32 i = ((afl->fsrv.real_map_size + 3) >> 2); u32 ret = 0; while (i--) { u32 v = *(ptr++); - if (!v) { continue; } + if (likely(!v)) { continue; } if (v & 0x000000ffU) { ++ret; } if (v & 0x0000ff00U) { ++ret; } if (v & 0x00ff0000U) { ++ret; } @@ -117,7 +117,7 @@ u32 count_bytes(afl_state_t *afl, u8 *mem) { u32 count_non_255_bytes(afl_state_t *afl, u8 *mem) { u32 *ptr = (u32 *)mem; - u32 i = (afl->fsrv.map_size >> 2); + u32 i = ((afl->fsrv.real_map_size + 3) >> 2); u32 ret = 0; while (i--) { @@ -127,7 +127,7 @@ u32 count_non_255_bytes(afl_state_t *afl, u8 *mem) { /* This is called on the virgin bitmap, so optimize for the most likely case. */ - if (v == 0xffffffffU) { continue; } + if (likely(v == 0xffffffffU)) { continue; } if ((v & 0x000000ffU) != 0x000000ffU) { ++ret; } if ((v & 0x0000ff00U) != 0x0000ff00U) { ++ret; } if ((v & 0x00ff0000U) != 0x00ff0000U) { ++ret; } @@ -216,14 +216,14 @@ inline u8 has_new_bits(afl_state_t *afl, u8 *virgin_map) { u64 *current = (u64 *)afl->fsrv.trace_bits; u64 *virgin = (u64 *)virgin_map; - u32 i = (afl->fsrv.map_size >> 3); + u32 i = ((afl->fsrv.real_map_size + 7) >> 3); #else u32 *current = (u32 *)afl->fsrv.trace_bits; u32 *virgin = (u32 *)virgin_map; - u32 i = (afl->fsrv.map_size >> 2); + u32 i = ((afl->fsrv.real_map_size + 3) >> 2); #endif /* ^WORD_SIZE_64 */ diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 426580d2..152bebe9 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -543,9 +543,11 @@ void show_stats(afl_state_t *afl) { FATAL( "Incorrect fuzzing setup detected. Your target seems to have loaded " - "incorrectly instrumented shared libraries. If you use LTO mode " + "incorrectly instrumented shared libraries (%u of %u/%u). If you use " + "LTO mode " "please see instrumentation/README.lto.md. To ignore this problem " - "and continue fuzzing just set 'AFL_IGNORE_PROBLEMS=1'.\n"); + "and continue fuzzing just set 'AFL_IGNORE_PROBLEMS=1'.\n", + t_bytes, afl->fsrv.real_map_size, afl->fsrv.map_size); } diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 236553ce..8cddcb32 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1435,9 +1435,9 @@ int main(int argc, char **argv_orig, char **envp) { if (!quiet_mode || collect_coverage) { if (!tcnt && !have_coverage) { FATAL("No instrumentation detected" cRST); } - OKF("Captured %u tuples (highest value %u, total values %llu) in " - "'%s'." cRST, - tcnt, highest, total, out_file); + OKF("Captured %u tuples (map size %u, highest value %u, total values %llu) " + "in '%s'." cRST, + tcnt, fsrv->real_map_size, highest, total, out_file); if (collect_coverage) OKF("A coverage of %u edges were achieved out of %u existing (%.02f%%) " "with %llu input files.", -- cgit 1.4.1 From e46adb0be7c5e1a0a1d49723678a9df20dfbe400 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 8 Dec 2021 11:28:33 +0100 Subject: fix AFL_REAL_LD for afl-cc --- docs/Changelog.md | 1 + src/afl-cc.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/docs/Changelog.md b/docs/Changelog.md index be5cac43..34b9affb 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -48,6 +48,7 @@ sending a mail to . - fixed a potential crash in targets for LAF string handling - added AFL_USE_TSAN thread sanitizer support - llvm and LTO mode modified to work with new llvm 14-dev (again) + - fix for AFL_REAL_LD - added the very good grammar mutator "GramaTron" to the custom_mutators - added optimin, a faster and better corpus minimizer by diff --git a/src/afl-cc.c b/src/afl-cc.c index 6bdb9572..1448d8ae 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -555,7 +555,8 @@ static void edit_params(u32 argc, char **argv, char **envp) { if (lto_mode && !have_c) { - u8 *ld_path = strdup(AFL_REAL_LD); + u8 *ld_path = NULL; + if (getenv("AFL_REAL_LD")) { ld_path = strdup(getenv("AFL_REAL_LD")); } if (!ld_path || !*ld_path) { ld_path = strdup("ld.lld"); } if (!ld_path) { PFATAL("Could not allocate mem for ld_path"); } #if defined(AFL_CLANG_LDPATH) && LLVM_MAJOR >= 12 -- cgit 1.4.1