From a632c00b0d023b8a40d09839fbb2662da1cb5d37 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 12 Jun 2020 16:08:49 +0200 Subject: switch to faster and better hash + random --- include/hash.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'include/hash.h') diff --git a/include/hash.h b/include/hash.h index cec51eac..09dabb59 100644 --- a/include/hash.h +++ b/include/hash.h @@ -30,9 +30,16 @@ #include "types.h" -#ifdef __x86_64__ +u32 hash32(const void *key, u32 len, u32 seed); +u64 hash64(const void *key, u32 len, u64 seed); - #define ROL64(_x, _r) ((((u64)(_x)) << (_r)) | (((u64)(_x)) >> (64 - (_r)))) +#if 0 + +The following code is disabled because xxh3 with a 32 bit resukt is 30% faster + + #ifdef __x86_64__ + + #define ROL64(_x, _r) ((((u64)(_x)) << (_r)) | (((u64)(_x)) >> (64 - (_r)))) static inline u32 hash32(const void *key, u32 len, u32 seed) { @@ -65,9 +72,9 @@ static inline u32 hash32(const void *key, u32 len, u32 seed) { } -#else + #else - #define ROL32(_x, _r) ((((u32)(_x)) << (_r)) | (((u32)(_x)) >> (32 - (_r)))) + #define ROL32(_x, _r) ((((u32)(_x)) << (_r)) | (((u32)(_x)) >> (32 - (_r)))) static inline u32 hash32(const void *key, u32 len, u32 seed) { @@ -100,7 +107,8 @@ static inline u32 hash32(const void *key, u32 len, u32 seed) { } -#endif /* ^__x86_64__ */ + #endif /* ^__x86_64__ */ +#endif #endif /* !_HAVE_HASH_H */ -- cgit 1.4.1 From 1542c7f49c00cd7d701869f951b9a2a126a7b960 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 13 Jun 2020 10:58:30 +0200 Subject: fix typos --- docs/Changelog.md | 27 ++++++++++++++------------- include/hash.h | 2 +- src/afl-performance.c | 20 ++++++++++++++------ 3 files changed, 29 insertions(+), 20 deletions(-) (limited to 'include/hash.h') diff --git a/docs/Changelog.md b/docs/Changelog.md index 751dd707..caf262fc 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -11,13 +11,13 @@ sending a mail to . ### Version ++2.65d (dev) - afl-fuzz: - - -S secondary nodes now only sync from the main node to increase performance, - the -M main node still syncs from everyone. Added checks that ensure - exactly one main node is present and warn otherwise - - If no main node is present at a sync one secondary node automatically becomes - a temporary main node until a real main nodes shows up - - switched murmur2 hashing and random() for xxh3 and xoshiro256**, giving up to 5.5% speed - increase + - -S secondary nodes now only sync from the main node to increase + performance, the -M main node still syncs from everyone. Added checks + that ensure exactly one main node is present and warn otherwise + - If no main node is present at a sync one secondary node automatically + becomes a temporary main node until a real main nodes shows up + - switched murmur2 hashing and random() for xxh3 and xoshiro256**, + resulting in an up to 5.5% speed increase - Resizing the window does not crash afl-fuzz anymore - fix/update to MOpt (thanks to arnow117) - added MOpt dictionary support from repo @@ -26,10 +26,10 @@ sending a mail to . better coverage. The original afl instrumentation can be set via AFL_LLVM_INSTRUMENT=AFL. This is automatically done when the WHITELIST feature is used. - - some targets want a ld variant for LD that is not gcc/clang but ld, added - afl-ld-lto to solve this - - lowered minimum required llvm version to 3.4 (except LLVMInsTrim, - which needs 3.8.0) + - some targets want a ld variant for LD that is not gcc/clang but ld, + added afl-ld-lto to solve this + - lowered minimum required llvm version to 3.4 (except LLVMInsTrim, which + needs 3.8.0) - WHITELIST feature now supports wildcards (thanks to sirmc) - small change to cmplog to make it work with current llvm 11-dev - added AFL_LLVM_LAF_ALL, sets all laf-intel settings @@ -41,6 +41,7 @@ sending a mail to . - enable snapshot lkm also for persistent mode - Unicornafl - Added powerPC support from unicorn/next + - rust bindings! - persistent mode shared memory testcase handover (instead of via files/stdin) - 10-100% performance increase - General support for 64 bit PowerPC, RiscV, Sparc etc. @@ -49,8 +50,8 @@ sending a mail to . the same second - added lots of dictionaries from oss-fuzz, go-fuzz and Jakub Wilk - added former post_library examples to examples/custom_mutators/ - - Dockerfile upgraded to Ubuntu 20.04 Focal and installing llvm 11 and gcc 10 - so afl-clang-lto can be build + - Dockerfile upgraded to Ubuntu 20.04 Focal and installing llvm 11 and + gcc 10 so afl-clang-lto can be build ### Version ++2.65c (release): diff --git a/include/hash.h b/include/hash.h index 09dabb59..6910e0e2 100644 --- a/include/hash.h +++ b/include/hash.h @@ -35,7 +35,7 @@ u64 hash64(const void *key, u32 len, u64 seed); #if 0 -The following code is disabled because xxh3 with a 32 bit resukt is 30% faster +The following code is disabled because xxh3 is 30% faster #ifdef __x86_64__ diff --git a/src/afl-performance.c b/src/afl-performance.c index 7a911ffd..28564eb8 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -1,10 +1,11 @@ -/* Written in 2019 by David Blackman and Sebastiano Vigna (vigna@acm.org) +/* + Written in 2019 by David Blackman and Sebastiano Vigna (vigna@acm.org) -To the extent possible under law, the author has dedicated all copyright -and related and neighboring rights to this software to the public domain -worldwide. This software is distributed without any warranty. + To the extent possible under law, the author has dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. -See . + See . This is xoshiro256++ 1.0, one of our all-purpose, rock-solid generators. It has excellent (sub-ns) speed, a state (256 bits) that is large @@ -15,13 +16,17 @@ See . The state must be seeded so that it is not everywhere zero. If you have a 64-bit seed, we suggest to seed a splitmix64 generator and use its - output to fill s. */ + output to fill s[]. +*/ #include #include "afl-fuzz.h" #include "types.h" #include "xxh3.h" +/* we use xoshiro256** instead of rand/random because it is 10x faster and has + better randomness properties. */ + static inline uint64_t rotl(const uint64_t x, int k) { return (x << k) | (x >> (64 - k)); @@ -122,6 +127,9 @@ void long_jump(afl_state_t *afl) { } +/* we switch from afl's murmur implementation to xxh3 as it is 30% faster - + and get 64 bit hashes instead of just 32 bit. Less collisions! :-) */ + u32 hash32(const void *key, u32 len, u32 seed) { return XXH64(key, len, seed) % 0x100000000; -- cgit 1.4.1 From c25a602a0370f484e32adbf186290d2504cf3f12 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sun, 28 Jun 2020 23:47:57 +0200 Subject: less problematic definitions --- examples/aflpp_driver/GNUmakefile | 10 +++++++--- examples/aflpp_driver/aflpp_driver.cpp | 2 +- examples/aflpp_driver/aflpp_driver_test.cpp | 2 +- include/afl-prealloc.h | 2 +- include/alloc-inl.h | 6 +++--- include/hash.h | 4 ++-- src/afl-performance.c | 10 +++++++--- 7 files changed, 22 insertions(+), 14 deletions(-) (limited to 'include/hash.h') diff --git a/examples/aflpp_driver/GNUmakefile b/examples/aflpp_driver/GNUmakefile index bd568224..a993c8a9 100644 --- a/examples/aflpp_driver/GNUmakefile +++ b/examples/aflpp_driver/GNUmakefile @@ -18,8 +18,11 @@ libAFLDriver.a: aflpp_driver.o ar ru libAFLDriver.a aflpp_driver.o debug: + $(LLVM_BINDIR)clang++ -Wno-deprecated -I../../include $(FLAGS) -D_DEBUG=\"1\" -c -o afl-performance.o ../../src/afl-performance.c $(LLVM_BINDIR)clang++ -I../../include -D_DEBUG=\"1\" -g -stdlib=libc++ -funroll-loops -std=c++11 -c aflpp_driver.cpp - ar ru libAFLDriver.a aflpp_driver.o + #$(LLVM_BINDIR)clang++ -S -emit-llvm -Wno-deprecated -I../../include $(FLAGS) -D_DEBUG=\"1\" -c -o afl-performance.ll ../../src/afl-performance.c + #$(LLVM_BINDIR)clang++ -S -emit-llvm -I../../include -D_DEBUG=\"1\" -g -stdlib=libc++ -funroll-loops -std=c++11 -c aflpp_driver.cpp + ar ru libAFLDriver.a afl-performance.o aflpp_driver.o aflpp_qemu_driver.o: aflpp_qemu_driver.c $(LLVM_BINDIR)clang $(FLAGS) -O0 -funroll-loops -c aflpp_qemu_driver.c @@ -33,8 +36,9 @@ aflpp_qemu_driver_hook.so: aflpp_qemu_driver_hook.o aflpp_qemu_driver_hook.o: aflpp_qemu_driver_hook.c $(LLVM_BINDIR)clang -fPIC $(FLAGS) -funroll-loops -c aflpp_qemu_driver_hook.c -test: libAFLDriver.a aflpp_driver_test.cpp - afl-clang-fast++ -I../../include -Wl,--allow-multiple-definition -stdlib=libc++ -funroll-loops -std=c++11 -o aflpp_driver_test aflpp_driver_test.cpp libAFLDriver.a +test: debug + #clang++ -S -emit-llvm -D_DEBUG=\"1\" -I../../include -Wl,--allow-multiple-definition -stdlib=libc++ -funroll-loops -std=c++11 -o aflpp_driver_test.ll aflpp_driver_test.cpp + afl-clang-fast++ -D_DEBUG=\"1\" -I../../include -Wl,--allow-multiple-definition -stdlib=libc++ -funroll-loops -std=c++11 -o aflpp_driver_test aflpp_driver_test.cpp libAFLDriver.a clean: rm -f *.o libAFLDriver*.a libAFLQemuDriver.a aflpp_qemu_driver_hook.so *~ core aflpp_driver_test diff --git a/examples/aflpp_driver/aflpp_driver.cpp b/examples/aflpp_driver/aflpp_driver.cpp index a1eab178..d6163bdf 100644 --- a/examples/aflpp_driver/aflpp_driver.cpp +++ b/examples/aflpp_driver/aflpp_driver.cpp @@ -277,7 +277,7 @@ int main(int argc, char **argv) { int num_runs = 0; while (__afl_persistent_loop(N)) { #ifdef _DEBUG - fprintf(stderr, "CLIENT crc: %08x len: %u\n", hash64(__afl_fuzz_ptr, *__afl_fuzz_len, 0xa5b35705), *__afl_fuzz_len); + fprintf(stderr, "CLIENT crc: %016llx len: %u\n", hash64(__afl_fuzz_ptr, *__afl_fuzz_len, 0xa5b35705), *__afl_fuzz_len); fprintf(stderr, "RECV:"); for (int i = 0; i < *__afl_fuzz_len; i++) fprintf(stderr, "%02x", __afl_fuzz_ptr[i]); diff --git a/examples/aflpp_driver/aflpp_driver_test.cpp b/examples/aflpp_driver/aflpp_driver_test.cpp index 799c743d..13dc09b9 100644 --- a/examples/aflpp_driver/aflpp_driver_test.cpp +++ b/examples/aflpp_driver/aflpp_driver_test.cpp @@ -5,7 +5,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { - fprintf(stderr, "FUNC crc: %08x len: %lu\n", hash32(Data, Size, 0xa5b35705), Size); + fprintf(stderr, "FUNC crc: %016llx len: %lu\n", hash64((u8*)Data, (unsigned int) Size, (unsigned long long int) 0xa5b35705), Size); if (Size < 5) return 0; diff --git a/include/afl-prealloc.h b/include/afl-prealloc.h index 5e5d7b85..a9de3ba2 100644 --- a/include/afl-prealloc.h +++ b/include/afl-prealloc.h @@ -60,7 +60,7 @@ typedef enum prealloc_status { \ if ((prealloc_counter) >= (prealloc_size)) { \ \ - el_ptr = malloc(sizeof(*el_ptr)); \ + el_ptr = (element_t *) malloc(sizeof(*el_ptr)); \ if (!el_ptr) { FATAL("error in list.h -> out of memory for element!"); } \ el_ptr->pre_status = PRE_STATUS_MALLOC; \ \ diff --git a/include/alloc-inl.h b/include/alloc-inl.h index ca593549..decc2d43 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -170,10 +170,10 @@ static inline u8 *DFL_ck_strdup(u8 *str) { size = strlen((char *)str) + 1; ALLOC_CHECK_SIZE(size); - ret = malloc(size); + ret = (u8*) malloc(size); ALLOC_CHECK_RESULT(ret, size); - return memcpy(ret, str, size); + return (u8*)memcpy(ret, str, size); } @@ -204,7 +204,7 @@ static inline u8 *DFL_ck_memdup_str(u8 *mem, u32 size) { if (!mem || !size) { return NULL; } ALLOC_CHECK_SIZE(size); - ret = malloc(size + 1); + ret = (u8*) malloc(size + 1); ALLOC_CHECK_RESULT(ret, size); memcpy(ret, mem, size); diff --git a/include/hash.h b/include/hash.h index 6910e0e2..9d42e44b 100644 --- a/include/hash.h +++ b/include/hash.h @@ -30,8 +30,8 @@ #include "types.h" -u32 hash32(const void *key, u32 len, u32 seed); -u64 hash64(const void *key, u32 len, u64 seed); +u32 hash32(u8 *key, u32 len, u32 seed); +u64 hash64(u8 *key, u32 len, u64 seed); #if 0 diff --git a/src/afl-performance.c b/src/afl-performance.c index a3febdbf..b3d30cbd 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -37,7 +37,7 @@ void rand_set_seed(afl_state_t *afl, s64 init_seed) { afl->init_seed = init_seed; afl->rand_seed[0] = - hash64((void *)&afl->init_seed, sizeof(afl->init_seed), HASH_CONST); + hash64((u8 *)&afl->init_seed, sizeof(afl->init_seed), HASH_CONST); afl->rand_seed[1] = afl->rand_seed[0] ^ 0x1234567890abcdef; afl->rand_seed[2] = afl->rand_seed[0] & 0x0123456789abcdef; afl->rand_seed[3] = afl->rand_seed[0] | 0x01abcde43f567908; @@ -141,13 +141,17 @@ void long_jump(afl_state_t *afl) { /* we switch from afl's murmur implementation to xxh3 as it is 30% faster - and get 64 bit hashes instead of just 32 bit. Less collisions! :-) */ -u32 inline hash32(const void *key, u32 len, u32 seed) { +u32 inline hash32(void *key, u32 len, u32 seed) { return (u32)XXH64(key, len, seed); } -u64 inline hash64(const void *key, u32 len, u64 seed) { +#ifdef _DEBUG +u64 hash64(u8 *key, u32 len, u64 seed) { +#else +u64 inline hash64(u8 *key, u32 len, u64 seed) { +#endif return XXH64(key, len, seed); -- cgit 1.4.1 From 6d0f086d9cb1b8eedbeb4c9654f9e44870460e8d Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sun, 28 Jun 2020 23:50:25 +0200 Subject: less problematic definitions --- include/hash.h | 2 +- src/afl-performance.c | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include/hash.h') diff --git a/include/hash.h b/include/hash.h index 9d42e44b..9319ab95 100644 --- a/include/hash.h +++ b/include/hash.h @@ -41,7 +41,7 @@ The following code is disabled because xxh3 is 30% faster #define ROL64(_x, _r) ((((u64)(_x)) << (_r)) | (((u64)(_x)) >> (64 - (_r)))) -static inline u32 hash32(const void *key, u32 len, u32 seed) { +static inline u32 hash32(u8 *key, u32 len, u32 seed) { const u64 *data = (u64 *)key; u64 h1 = seed ^ len; diff --git a/src/afl-performance.c b/src/afl-performance.c index b3d30cbd..0832dc39 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -141,7 +141,11 @@ void long_jump(afl_state_t *afl) { /* we switch from afl's murmur implementation to xxh3 as it is 30% faster - and get 64 bit hashes instead of just 32 bit. Less collisions! :-) */ -u32 inline hash32(void *key, u32 len, u32 seed) { +#ifdef _DEBUG +u32 hash32(u8 *key, u32 len, u32 seed) { +#else +u32 inline hash32(u8 *key, u32 len, u32 seed) { +#endif return (u32)XXH64(key, len, seed); -- cgit 1.4.1