From 6d23df2c7c5246eb2e3da393b99a9c06bac972c1 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 15 May 2023 17:13:20 +0200 Subject: add target_intelligence --- utils/target_intelligence/README.md | 61 +++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 utils/target_intelligence/README.md (limited to 'utils') diff --git a/utils/target_intelligence/README.md b/utils/target_intelligence/README.md new file mode 100644 index 00000000..086c9e20 --- /dev/null +++ b/utils/target_intelligence/README.md @@ -0,0 +1,61 @@ +# Target Intelligence + +These are some ideas you can do so that your target that you are fuzzing can +give helpful feedback to AFL++. + +## Add to the AFL++ dictionary from your target + +For this you target must be compiled for CMPLOG (`AFL_LLVM_CMPLOG=1`). + +Add in your source code: + +``` +__attribute__((weak)) void __cmplog_rtn_hook_strn(u8 *ptr1, u8 *ptr2, u64 len); +__attribute__((weak)) void __cmplog_ins_hook1(uint8_t arg1, uint8_t arg2, uint8_t attr); +__attribute__((weak)) void __cmplog_ins_hook2(uint16_t arg1, uint16_t arg2, uint8_t attr); +__attribute__((weak)) void __cmplog_ins_hook4(uint32_t arg1, uint32_t arg2, uint8_t attr); +__attribute__((weak)) void __cmplog_ins_hook8(uint64_t arg1, uint64_t arg2, uint8_t attr); + +int in_your_function(...) { + + // to add two strings to the AFL++ dictionary: + if (__cmplog_rtn_hook_strn) + __cmplog_rtn_hook_strn(string1, length_of_string1, string2, length_of_string2); + + // to add two 32 bit integers to the AFL++ dictionary: + if (__cmplog_ins_hook4) + __cmplog_ins_hook4(first_32_bit_var, second_32_bit_var, 0); + +} +``` + +Note that this only makes sense if these values are in-depth processed in the +target in a way that AFL++ CMPLOG cannot uncover these, e.g. if these values +are transformed by a matrix computation. + +Fixed values are always better to give to afl-fuzz via a `-x dictionary`. + +## Add inputs to AFL++ dictionary from your target + +If for whatever reason you want your target to propose new inputs to AFL++, +then this is actually very easy. +The environment variable `AFL_CUSTOM_INFO_OUT` contains the output directory +of this run - including the fuzzer instance name (e.g. `default`), so if you +run `afl-fuzz -o out -S foobar`, the value would be `out/foobar`). + +To show afl-fuzz an input it should consider just do the following: + +1. create the directory `$AFL_CUSTOM_INFO_OUT/../target/queue` +2. create any new inputs you want afl-fuzz to notice in that directory with the + following naming convention: `id:NUMBER-OF-LENGTH-SIX-WITH-LEADING-ZEROES,whatever` + where that number has to be increasing. + e.g.: +``` + id:000000,first_file + id:000001,second_file + id:000002,third_file + etc. +``` + +Note that this will not work in nyx_mode because afl-fuzz cannot see inside the +virtual machine. -- cgit 1.4.1 From 029e039cbcbf9e30f35eb255162679b8d609d25d Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 21 May 2023 17:49:14 +0200 Subject: code format --- .custom-format.py | 2 +- frida_mode/src/lib/lib_apple.c | 4 +-- include/config.h | 2 +- instrumentation/SanitizerCoverageLTO.so.cc | 4 +-- instrumentation/SanitizerCoveragePCGUARD.so.cc | 4 +-- instrumentation/afl-compiler-rt.o.c | 14 +++++----- instrumentation/afl-llvm-lto-instrumentlist.so.cc | 2 +- instrumentation/cmplog-routines-pass.cc | 4 +-- instrumentation/compare-transform-pass.so.cc | 2 +- instrumentation/split-compares-pass.so.cc | 2 +- qemu_mode/libcompcov/libcompcov.so.c | 6 ++++- src/afl-cc.c | 8 +++--- src/afl-fuzz-redqueen.c | 32 +++++++++++------------ utils/afl_untracer/afl-untracer.c | 2 +- utils/libtokencap/libtokencap.so.c | 6 ++++- utils/socket_fuzzing/socketfuzz.c | 3 ++- 16 files changed, 53 insertions(+), 44 deletions(-) (limited to 'utils') diff --git a/.custom-format.py b/.custom-format.py index 1295ce55..1d5c8839 100755 --- a/.custom-format.py +++ b/.custom-format.py @@ -24,7 +24,7 @@ import importlib.metadata # string_re = re.compile('(\\"(\\\\.|[^"\\\\])*\\")') # TODO: for future use -CURRENT_LLVM = os.getenv('LLVM_VERSION', 14) +CURRENT_LLVM = os.getenv('LLVM_VERSION', 15) CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN", "") diff --git a/frida_mode/src/lib/lib_apple.c b/frida_mode/src/lib/lib_apple.c index 634e0e30..d29d0303 100644 --- a/frida_mode/src/lib/lib_apple.c +++ b/frida_mode/src/lib/lib_apple.c @@ -17,8 +17,8 @@ static gboolean lib_get_main_module(const GumModuleDetails *details, GumDarwinModule **ret = (GumDarwinModule **)user_data; GumDarwinModule *module = gum_darwin_module_new_from_memory( - details->path, mach_task_self(), details->range->base_address, - GUM_DARWIN_MODULE_FLAGS_NONE, NULL); + details->path, mach_task_self(), details->range->base_address, + GUM_DARWIN_MODULE_FLAGS_NONE, NULL); FVERBOSE("Found main module: %s", module->name); diff --git a/include/config.h b/include/config.h index 764c29dc..194786f7 100644 --- a/include/config.h +++ b/include/config.h @@ -81,7 +81,7 @@ will be kept and written to the crash/ directory as RECORD:... files. Note that every crash will be written, not only unique ones! */ -//#define AFL_PERSISTENT_RECORD +// #define AFL_PERSISTENT_RECORD /* console output colors: There are three ways to configure its behavior * 1. default: colored outputs fixed on: defined USE_COLOR && defined diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index b3b0d2cd..d7b03634 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -1478,8 +1478,8 @@ GlobalVariable *ModuleSanitizerCoverageLTO::CreateFunctionLocalArrayInSection( ArrayType *ArrayTy = ArrayType::get(Ty, NumElements); auto Array = new GlobalVariable( - *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, - Constant::getNullValue(ArrayTy), "__sancov_gen_"); + *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, + Constant::getNullValue(ArrayTy), "__sancov_gen_"); #if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 41c38283..8fed2042 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -714,8 +714,8 @@ GlobalVariable *ModuleSanitizerCoverageAFL::CreateFunctionLocalArrayInSection( ArrayType *ArrayTy = ArrayType::get(Ty, NumElements); auto Array = new GlobalVariable( - *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, - Constant::getNullValue(ArrayTy), "__sancov_gen_"); + *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, + Constant::getNullValue(ArrayTy), "__sancov_gen_"); #if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 5372fae0..3f8b519b 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -544,12 +544,12 @@ static void __afl_map_shm(void) { if (__afl_map_size && __afl_map_size > MAP_SIZE) { - u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); - if (!map_env || atoi((char *)map_env) < MAP_SIZE) { + u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); + if (!map_env || atoi((char *)map_env) < MAP_SIZE) { - fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); - send_forkserver_error(FS_ERROR_MAP_SIZE); - _exit(1); + fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); + send_forkserver_error(FS_ERROR_MAP_SIZE); + _exit(1); } @@ -561,13 +561,13 @@ static void __afl_map_shm(void) { if (!__afl_area_ptr || __afl_area_ptr == (void *)-1) { - if (__afl_map_addr) + if (__afl_map_addr) send_forkserver_error(FS_ERROR_MAP_ADDR); else send_forkserver_error(FS_ERROR_SHMAT); perror("shmat for map"); - _exit(1); + _exit(1); } diff --git a/instrumentation/afl-llvm-lto-instrumentlist.so.cc b/instrumentation/afl-llvm-lto-instrumentlist.so.cc index db5bd55e..61f97d77 100644 --- a/instrumentation/afl-llvm-lto-instrumentlist.so.cc +++ b/instrumentation/afl-llvm-lto-instrumentlist.so.cc @@ -45,7 +45,7 @@ #include "llvm/IR/Module.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" -//#include "llvm/Transforms/IPO/PassManagerBuilder.h" +// #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/IR/PassManager.h" diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index 39db5aa4..c3fbed8d 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -542,7 +542,7 @@ bool CmpLogRoutines::hookRtns(Module &M) { Value *v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); Value *v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); Value *v3Pbitcast = IRB.CreateBitCast( - v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); + v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); Value *v3Pcasted = IRB.CreateIntCast(v3Pbitcast, IntegerType::get(C, 64), false); args.push_back(v1Pcasted); @@ -608,7 +608,7 @@ bool CmpLogRoutines::hookRtns(Module &M) { Value *v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); Value *v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); Value *v3Pbitcast = IRB.CreateBitCast( - v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); + v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); Value *v3Pcasted = IRB.CreateIntCast(v3Pbitcast, IntegerType::get(C, 64), false); args.push_back(v1Pcasted); diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index efc99d20..5dd705cf 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -623,7 +623,7 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, IRBuilder<> cur_lenchk_IRB(&*(cur_lenchk_bb->getFirstInsertionPt())); Value *icmp = cur_lenchk_IRB.CreateICmpEQ( - sizedValue, ConstantInt::get(sizedValue->getType(), i)); + sizedValue, ConstantInt::get(sizedValue->getType(), i)); cur_lenchk_IRB.CreateCondBr(icmp, end_bb, cur_cmp_bb); cur_lenchk_bb->getTerminator()->eraseFromParent(); diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index 8a07610c..aec6758e 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -60,7 +60,7 @@ using namespace llvm; // uncomment this toggle function verification at each step. horribly slow, but // helps to pinpoint a potential problem in the splitting code. -//#define VERIFY_TOO_MUCH 1 +// #define VERIFY_TOO_MUCH 1 namespace { diff --git a/qemu_mode/libcompcov/libcompcov.so.c b/qemu_mode/libcompcov/libcompcov.so.c index b6ee0019..b57e9701 100644 --- a/qemu_mode/libcompcov/libcompcov.so.c +++ b/qemu_mode/libcompcov/libcompcov.so.c @@ -68,7 +68,11 @@ static int debug_fd = -1; #define MAX_MAPPINGS 1024 -static struct mapping { void *st, *en; } __compcov_ro[MAX_MAPPINGS]; +static struct mapping { + + void *st, *en; + +} __compcov_ro[MAX_MAPPINGS]; static u32 __compcov_ro_cnt; diff --git a/src/afl-cc.c b/src/afl-cc.c index 972ac8cd..e3cc04dd 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -933,10 +933,10 @@ static void edit_params(u32 argc, char **argv, char **envp) { } - //#if LLVM_MAJOR >= 13 - // // Use the old pass manager in LLVM 14 which the AFL++ passes still - // use. cc_params[cc_par_cnt++] = "-flegacy-pass-manager"; - //#endif + // #if LLVM_MAJOR >= 13 + // // Use the old pass manager in LLVM 14 which the AFL++ passes still + // use. cc_params[cc_par_cnt++] = "-flegacy-pass-manager"; + // #endif if (lto_mode && !have_c) { diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 6e4a655b..d9dc50df 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -28,8 +28,8 @@ #include "afl-fuzz.h" #include "cmplog.h" -//#define _DEBUG -//#define CMPLOG_INTROSPECTION +// #define _DEBUG +// #define CMPLOG_INTROSPECTION // CMP attribute enum enum { @@ -571,7 +571,7 @@ static u8 its_fuzz(afl_state_t *afl, u8 *buf, u32 len, u8 *status) { } -//#ifdef CMPLOG_SOLVE_TRANSFORM +// #ifdef CMPLOG_SOLVE_TRANSFORM static int strntoll(const char *str, size_t sz, char **end, int base, long long *out) { @@ -771,7 +771,7 @@ static void to_base64(u8 *src, u8 *dst, u32 dst_len) { #endif -//#endif +// #endif static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u64 pattern, u64 repl, u64 o_pattern, @@ -803,8 +803,8 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, // o_pattern, pattern, repl, changed_val, idx, taint_len, // hshape, attr); - //#ifdef CMPLOG_SOLVE_TRANSFORM - // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3 + // #ifdef CMPLOG_SOLVE_TRANSFORM + // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3 if (afl->cmplog_enable_transform && (lvl & LVL3)) { u8 *endptr; @@ -1120,7 +1120,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - //#endif + // #endif // we only allow this for ascii2integer (above) so leave if this is the case if (unlikely(pattern == o_pattern)) { return 0; } @@ -1275,7 +1275,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, // 16 = modified float, 32 = modified integer (modified = wont match // in original buffer) - //#ifdef CMPLOG_SOLVE_ARITHMETIC + // #ifdef CMPLOG_SOLVE_ARITHMETIC if (!afl->cmplog_enable_arith || lvl < LVL3 || attr == IS_TRANSFORM) { return 0; @@ -1440,8 +1440,8 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - //#endif /* - // CMPLOG_SOLVE_ARITHMETIC + // #endif /* + // CMPLOG_SOLVE_ARITHMETIC return 0; @@ -1948,9 +1948,9 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, #ifndef CMPLOG_COMBINE (void)(cbuf); #endif - //#ifndef CMPLOG_SOLVE_TRANSFORM - // (void)(changed_val); - //#endif + // #ifndef CMPLOG_SOLVE_TRANSFORM + // (void)(changed_val); + // #endif if (afl->fsrv.total_execs - last_update > screen_update) { @@ -2418,7 +2418,7 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - //#endif + // #endif return 0; @@ -2818,9 +2818,9 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { } else if ((lvl & LVL1) - //#ifdef CMPLOG_SOLVE_TRANSFORM + // #ifdef CMPLOG_SOLVE_TRANSFORM || ((lvl & LVL3) && afl->cmplog_enable_transform) - //#endif + // #endif ) { if (unlikely(rtn_fuzz(afl, k, orig_buf, buf, cbuf, len, lvl, taint))) { diff --git a/utils/afl_untracer/afl-untracer.c b/utils/afl_untracer/afl-untracer.c index a18e314e..e1038212 100644 --- a/utils/afl_untracer/afl-untracer.c +++ b/utils/afl_untracer/afl-untracer.c @@ -288,7 +288,7 @@ library_list_t *find_library(char *name) { #pragma GCC optimize("O0") void breakpoint(void) { - if (debug) fprintf(stderr, "Breakpoint function \"breakpoint\" reached.\n"); + if (debug) fprintf(stderr, "Breakpoint function \"breakpoint\" reached.\n"); } diff --git a/utils/libtokencap/libtokencap.so.c b/utils/libtokencap/libtokencap.so.c index 299056ab..b21f3068 100644 --- a/utils/libtokencap/libtokencap.so.c +++ b/utils/libtokencap/libtokencap.so.c @@ -81,7 +81,11 @@ void *(*__libc_memmem)(const void *haystack, size_t haystack_len, #define MAX_MAPPINGS 1024 -static struct mapping { void *st, *en; } __tokencap_ro[MAX_MAPPINGS]; +static struct mapping { + + void *st, *en; + +} __tokencap_ro[MAX_MAPPINGS]; static u32 __tokencap_ro_cnt; static u8 __tokencap_ro_loaded; diff --git a/utils/socket_fuzzing/socketfuzz.c b/utils/socket_fuzzing/socketfuzz.c index 3ec8383b..7497519e 100644 --- a/utils/socket_fuzzing/socketfuzz.c +++ b/utils/socket_fuzzing/socketfuzz.c @@ -23,7 +23,8 @@ #include #include #include -//#include "logging.h" // switche from preeny_info() to fprintf(stderr, "Info: " +// #include "logging.h" // switched from preeny_info() to fprintf(stderr, "Info: +// " // // originals -- cgit 1.4.1 From 450e00446dfe23a74b08c4ad88b082d48e1f0b66 Mon Sep 17 00:00:00 2001 From: cuanduo Date: Fri, 16 Jun 2023 08:28:05 +0800 Subject: fix bug --- utils/afl_network_proxy/afl-network-server.c | 1 + 1 file changed, 1 insertion(+) (limited to 'utils') diff --git a/utils/afl_network_proxy/afl-network-server.c b/utils/afl_network_proxy/afl-network-server.c index 04309ada..7eb3d18e 100644 --- a/utils/afl_network_proxy/afl-network-server.c +++ b/utils/afl_network_proxy/afl-network-server.c @@ -173,6 +173,7 @@ static void set_up_environment(afl_forkserver_t *fsrv) { } out_file = alloc_printf("%s/.afl-input-temp-%u", use_dir, getpid()); + fsrv->out_file = out_file; } -- cgit 1.4.1 From 90f83c13d08f44fbf50036076a1772909c4d2c86 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 22 Jun 2023 09:24:00 +0200 Subject: remove dead code, code format --- .custom-format.py | 2 +- docs/Changelog.md | 3 ++ include/alloc-inl.h | 8 +++--- instrumentation/SanitizerCoveragePCGUARD.so.cc | 39 ++------------------------ qemu_mode/libqasan/dlmalloc.c | 2 +- src/afl-fuzz-init.c | 8 +++--- src/afl-fuzz.c | 3 +- utils/afl_network_proxy/afl-network-server.c | 2 +- 8 files changed, 19 insertions(+), 48 deletions(-) (limited to 'utils') diff --git a/.custom-format.py b/.custom-format.py index 1d5c8839..3521c05d 100755 --- a/.custom-format.py +++ b/.custom-format.py @@ -24,7 +24,7 @@ import importlib.metadata # string_re = re.compile('(\\"(\\\\.|[^"\\\\])*\\")') # TODO: for future use -CURRENT_LLVM = os.getenv('LLVM_VERSION', 15) +CURRENT_LLVM = os.getenv('LLVM_VERSION', 16) CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN", "") diff --git a/docs/Changelog.md b/docs/Changelog.md index 246c3cac..c850c43e 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -18,6 +18,9 @@ - fixed a bug inherited from vanilla AFL where a coverage of map[123] = 11 would be the same as map[1123] = 1 - warn on crashing inputs + - afl-cc + - fixed an off-by-one instrumentation of iselect, hurting coverage a bit. + Thanks to @amykweon for spotting and fixing! ### Version ++4.07c (release) diff --git a/include/alloc-inl.h b/include/alloc-inl.h index 1e9a192b..cff808b2 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -322,7 +322,7 @@ static inline void DFL_ck_free(void *mem) { static inline void *DFL_ck_realloc(void *orig, u32 size) { void *ret; - u32 old_size = 0; + u32 old_size = 0; if (!size) { @@ -392,7 +392,7 @@ static inline void *DFL_ck_realloc(void *orig, u32 size) { static inline u8 *DFL_ck_strdup(u8 *str) { void *ret; - u32 size; + u32 size; if (!str) return NULL; @@ -438,14 +438,14 @@ struct TRK_obj { void *ptr; char *file, *func; - u32 line; + u32 line; }; #ifdef AFL_MAIN struct TRK_obj *TRK[ALLOC_BUCKETS]; -u32 TRK_cnt[ALLOC_BUCKETS]; +u32 TRK_cnt[ALLOC_BUCKETS]; #define alloc_report() TRK_report() diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index d87af775..57b5d128 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -225,49 +225,18 @@ llvmGetPassPluginInfo() { } -#if LLVM_VERSION_MAJOR == 1 +#if LLVM_VERSION_MAJOR >= 16 PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, ModuleAnalysisManager &MAM) { - ModuleSanitizerCoverageAFL ModuleSancov(Options); - auto &FAM = MAM.getResult(M).getManager(); - auto DTCallback = [&FAM](Function &F) -> const DominatorTree *{ - - return &FAM.getResult(F); - - }; - - auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree * { - - return &FAM.getResult(F); - - }; - - if (!ModuleSancov.instrumentModule(M, DTCallback, PDTCallback)) - return PreservedAnalyses::all(); - - PreservedAnalyses PA = PreservedAnalyses::none(); - // GlobalsAA is considered stateless and does not get invalidated unless - // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers - // make changes that require GlobalsAA to be invalidated. - PA.abandon(); - return PA; - -} - #else - #if LLVM_VERSION_MAJOR >= 16 -PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, - ModuleAnalysisManager &MAM) { - - #else PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, ModuleAnalysisManager &MAM) { - #endif +#endif ModuleSanitizerCoverageAFL ModuleSancov(Options); auto &FAM = MAM.getResult(M).getManager(); - auto DTCallback = [&FAM](Function &F) -> const DominatorTree * { + auto DTCallback = [&FAM](Function &F) -> const DominatorTree *{ return &FAM.getResult(F); @@ -285,8 +254,6 @@ PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, } -#endif - std::pair ModuleSanitizerCoverageAFL::CreateSecStartEnd( Module &M, const char *Section, Type *Ty) { diff --git a/qemu_mode/libqasan/dlmalloc.c b/qemu_mode/libqasan/dlmalloc.c index 5d0b65ce..b459eb7b 100644 --- a/qemu_mode/libqasan/dlmalloc.c +++ b/qemu_mode/libqasan/dlmalloc.c @@ -1762,7 +1762,7 @@ static FORCEINLINE void *win32direct_mmap(size_t size) { static FORCEINLINE int win32munmap(void *ptr, size_t size) { MEMORY_BASIC_INFORMATION minfo; - char *cptr = (char *)ptr; + char *cptr = (char *)ptr; while (size) { diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 13802f40..24fd7077 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -1542,8 +1542,8 @@ double get_runnable_processes(void) { processes well. */ FILE *f = fopen("/proc/stat", "r"); - u8 tmp[1024]; - u32 val = 0; + u8 tmp[1024]; + u32 val = 0; if (!f) { return 0; } @@ -2226,7 +2226,7 @@ void check_crash_handling(void) { *BSD, so we can just let it slide for now. */ s32 fd = open("/proc/sys/kernel/core_pattern", O_RDONLY); - u8 fchar; + u8 fchar; if (fd < 0) { return; } @@ -2365,7 +2365,7 @@ void check_cpu_governor(afl_state_t *afl) { FATAL("Suboptimal CPU scaling governor"); #elif defined __APPLE__ - u64 min = 0, max = 0; + u64 min = 0, max = 0; size_t mlen = sizeof(min); if (afl->afl_env.afl_skip_cpufreq) return; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 8cf786af..79b05da7 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -164,7 +164,8 @@ static void usage(u8 *argv0, int more_help) { "\n" "Mutator settings:\n" - " -a - target expects ascii text input (prefer text mutators)\n" + " -a - target expects ascii text input (prefer text " + "mutators)\n" " -g minlength - set min length of generated fuzz input (default: 1)\n" " -G maxlength - set max length of generated fuzz input (default: " "%lu)\n" diff --git a/utils/afl_network_proxy/afl-network-server.c b/utils/afl_network_proxy/afl-network-server.c index 7eb3d18e..95b0a551 100644 --- a/utils/afl_network_proxy/afl-network-server.c +++ b/utils/afl_network_proxy/afl-network-server.c @@ -173,7 +173,7 @@ static void set_up_environment(afl_forkserver_t *fsrv) { } out_file = alloc_printf("%s/.afl-input-temp-%u", use_dir, getpid()); - fsrv->out_file = out_file; + fsrv->out_file = out_file; } -- cgit 1.4.1 From 2c40fc4ae8fe59580b13fa1e7dffa04c65bd6ae4 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 2 Sep 2023 10:04:14 +0000 Subject: afl untracer haiku build fix. --- utils/afl_untracer/Makefile | 7 ++++++- utils/afl_untracer/afl-untracer.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) (limited to 'utils') diff --git a/utils/afl_untracer/Makefile b/utils/afl_untracer/Makefile index 14a09b41..264aebe5 100644 --- a/utils/afl_untracer/Makefile +++ b/utils/afl_untracer/Makefile @@ -3,11 +3,16 @@ ifdef DEBUG else OPT=-O3 endif +SYS = $(shell uname -s) +DL = +ifeq "$(SYS)" "Linux" + DL = -ldl +endif all: afl-untracer libtestinstr.so afl-untracer: afl-untracer.c - $(CC) $(OPT) -I../../include -g -o afl-untracer afl-untracer.c -ldl + $(CC) $(OPT) -I../../include -g -o afl-untracer afl-untracer.c $(DL) libtestinstr.so: libtestinstr.c $(CC) -g -O0 -fPIC -o libtestinstr.so -shared libtestinstr.c diff --git a/utils/afl_untracer/afl-untracer.c b/utils/afl_untracer/afl-untracer.c index e1038212..5a67b996 100644 --- a/utils/afl_untracer/afl-untracer.c +++ b/utils/afl_untracer/afl-untracer.c @@ -53,7 +53,9 @@ #include #include +#if !defined(__HAIKU__) #include +#endif #include #include @@ -66,6 +68,9 @@ #include #include #include +#elif defined(__HAIKU__) + #include + #include #else #error "Unsupported platform" #endif @@ -231,7 +236,28 @@ void read_library_information(void) { start += size; } +#elif defined(__HAIKU__) + image_info ii; + int32 c = 0; + + while (get_next_image_info(0, &c, &ii) == B_OK) { + + liblist[liblist_cnt].name = (u8 *)strdup(ii.name); + liblist[liblist_cnt].addr_start = (u64)ii.text; + liblist[liblist_cnt].addr_end = (u64)((char *)ii.text + ii.text_size); + + if (debug) { + fprintf(stderr, "%s:%lx (%lx-%lx)\n", liblist[liblist_cnt].name, + (unsigned long)(liblist[liblist_cnt].addr_end - + liblist[liblist_cnt].addr_start), + (unsigned long)liblist[liblist_cnt].addr_start, + (unsigned long)(liblist[liblist_cnt].addr_end - 1)); + + } + + liblist_cnt++; + } #endif } @@ -655,6 +681,9 @@ static void sigtrap_handler(int signum, siginfo_t *si, void *context) { #elif defined(__FreeBSD__) && defined(__LP64__) ctx->uc_mcontext.mc_rip -= 1; addr = ctx->uc_mcontext.mc_rip; +#elif defined(__HAIKU__) && defined(__x86_64__) + ctx->uc_mcontext.rip -= 1; + addr = ctx->uc_mcontext.rip; #else #error "Unsupported platform" #endif -- cgit 1.4.1 From 3bae404733e27b3ec7769ad6d5d997dcd9ec6fa3 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 3 Sep 2023 11:25:03 +0200 Subject: code format --- frida_mode/src/instrument/instrument_arm64.c | 14 +++++--------- utils/afl_untracer/afl-untracer.c | 29 +++++++++++++++------------- 2 files changed, 21 insertions(+), 22 deletions(-) (limited to 'utils') diff --git a/frida_mode/src/instrument/instrument_arm64.c b/frida_mode/src/instrument/instrument_arm64.c index a0c66697..1147275f 100644 --- a/frida_mode/src/instrument/instrument_arm64.c +++ b/frida_mode/src/instrument/instrument_arm64.c @@ -402,17 +402,13 @@ bool instrument_write_inline(GumArm64Writer *cw, GumAddress code_addr, } - /* - * The mov instruction supports up to a 16-bit offset. If our offset is out of - * range, then it can end up clobbering the op-code portion of the instruction - * rather than just the operands. So return false and fall back to the + /* + * The mov instruction supports up to a 16-bit offset. If our offset is out of + * range, then it can end up clobbering the op-code portion of the instruction + * rather than just the operands. So return false and fall back to the * alternative instrumentation. */ - if (area_offset > UINT16_MAX) { - - return false; - - } + if (area_offset > UINT16_MAX) { return false; } code.code.mov_x0_curr_loc |= area_offset << 5; diff --git a/utils/afl_untracer/afl-untracer.c b/utils/afl_untracer/afl-untracer.c index 5a67b996..0e3f8a45 100644 --- a/utils/afl_untracer/afl-untracer.c +++ b/utils/afl_untracer/afl-untracer.c @@ -54,7 +54,7 @@ #include #if !defined(__HAIKU__) -#include + #include #endif #include #include @@ -236,28 +236,31 @@ void read_library_information(void) { start += size; } + #elif defined(__HAIKU__) image_info ii; - int32 c = 0; + int32 c = 0; while (get_next_image_info(0, &c, &ii) == B_OK) { - liblist[liblist_cnt].name = (u8 *)strdup(ii.name); - liblist[liblist_cnt].addr_start = (u64)ii.text; - liblist[liblist_cnt].addr_end = (u64)((char *)ii.text + ii.text_size); + liblist[liblist_cnt].name = (u8 *)strdup(ii.name); + liblist[liblist_cnt].addr_start = (u64)ii.text; + liblist[liblist_cnt].addr_end = (u64)((char *)ii.text + ii.text_size); - if (debug) { + if (debug) { - fprintf(stderr, "%s:%lx (%lx-%lx)\n", liblist[liblist_cnt].name, - (unsigned long)(liblist[liblist_cnt].addr_end - - liblist[liblist_cnt].addr_start), - (unsigned long)liblist[liblist_cnt].addr_start, - (unsigned long)(liblist[liblist_cnt].addr_end - 1)); + fprintf(stderr, "%s:%lx (%lx-%lx)\n", liblist[liblist_cnt].name, + (unsigned long)(liblist[liblist_cnt].addr_end - + liblist[liblist_cnt].addr_start), + (unsigned long)liblist[liblist_cnt].addr_start, + (unsigned long)(liblist[liblist_cnt].addr_end - 1)); - } + } + + liblist_cnt++; - liblist_cnt++; } + #endif } -- cgit 1.4.1 From 762fe0aad860c0c0fa3c9e19103c3df799498c72 Mon Sep 17 00:00:00 2001 From: Thomas Rooijakkers Date: Wed, 20 Sep 2023 11:00:17 +0200 Subject: forgot removal of compiler flag --- utils/qbdi_mode/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'utils') diff --git a/utils/qbdi_mode/build.sh b/utils/qbdi_mode/build.sh index 29fe0ee4..a92d81bd 100755 --- a/utils/qbdi_mode/build.sh +++ b/utils/qbdi_mode/build.sh @@ -52,6 +52,6 @@ ${compiler_prefix}${CC} -shared -o libdemo.so demo-so.c -w -g echo "[+] Building afl-fuzz for Android" # build afl-fuzz cd ../.. -${compiler_prefix}${CC} -DANDROID_DISABLE_FANCY=1 -O3 -funroll-loops -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign -I include/ -DAFL_PATH=\"/usr/local/lib/afl\" -DBIN_PATH=\"/usr/local/bin\" -DDOC_PATH=\"/usr/local/share/doc/afl\" -Wno-unused-function src/afl-fuzz*.c src/afl-common.c src/afl-sharedmem.c src/afl-forkserver.c src/afl-performance.c -o utils/qbdi_mode/afl-fuzz -ldl -lm -w +${compiler_prefix}${CC} -O3 -funroll-loops -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign -I include/ -DAFL_PATH=\"/usr/local/lib/afl\" -DBIN_PATH=\"/usr/local/bin\" -DDOC_PATH=\"/usr/local/share/doc/afl\" -Wno-unused-function src/afl-fuzz*.c src/afl-common.c src/afl-sharedmem.c src/afl-forkserver.c src/afl-performance.c -o utils/qbdi_mode/afl-fuzz -ldl -lm -w echo "[+] All done. Enjoy!" -- cgit 1.4.1 From f3d2127fd815bed2ec9dfab981123898d11cea65 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 6 Nov 2023 10:13:59 +0100 Subject: clang-format 16->17 --- .custom-format.py | 2 +- frida_mode/src/main.c | 8 +++--- include/afl-mutations.h | 2 +- include/xxhash.h | 30 +++++++++++------------ instrumentation/cmplog-instructions-pass.cc | 2 +- instrumentation/cmplog-routines-pass.cc | 2 +- instrumentation/cmplog-switches-pass.cc | 2 +- instrumentation/split-switches-pass.so.cc | 2 +- qemu_mode/libqasan/dlmalloc.c | 38 ++++++++++++++--------------- qemu_mode/libqasan/malloc.c | 4 +-- src/afl-fuzz-one.c | 30 +++++++++++------------ src/afl-fuzz-redqueen.c | 2 +- utils/libtokencap/libtokencap.so.c | 6 ++--- 13 files changed, 65 insertions(+), 65 deletions(-) (limited to 'utils') diff --git a/.custom-format.py b/.custom-format.py index 3521c05d..c8075ace 100755 --- a/.custom-format.py +++ b/.custom-format.py @@ -24,7 +24,7 @@ import importlib.metadata # string_re = re.compile('(\\"(\\\\.|[^"\\\\])*\\")') # TODO: for future use -CURRENT_LLVM = os.getenv('LLVM_VERSION', 16) +CURRENT_LLVM = os.getenv('LLVM_VERSION', 17) CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN", "") diff --git a/frida_mode/src/main.c b/frida_mode/src/main.c index f11c4b25..bd7b1351 100644 --- a/frida_mode/src/main.c +++ b/frida_mode/src/main.c @@ -49,10 +49,10 @@ extern void __libc_init(void *raw_args, void (*onexit)(void) __unused, int (*slingshot)(int, char **, char **), structors_array_t const *const structors); #else -extern int __libc_start_main(int (*main)(int, char **, char **), int argc, - char **ubp_av, void (*init)(void), - void (*fini)(void), void (*rtld_fini)(void), - void(*stack_end)); +extern int __libc_start_main(int (*main)(int, char **, char **), int argc, + char **ubp_av, void (*init)(void), + void (*fini)(void), void (*rtld_fini)(void), + void(*stack_end)); #endif typedef int (*main_fn_t)(int argc, char **argv, char **envp); diff --git a/include/afl-mutations.h b/include/afl-mutations.h index 98ba6fcf..d709b90d 100644 --- a/include/afl-mutations.h +++ b/include/afl-mutations.h @@ -1854,7 +1854,7 @@ inline u32 afl_mutate(afl_state_t *afl, u8 *buf, u32 len, u32 steps, for (u32 step = 0; step < steps; ++step) { - retry_havoc_step : { + retry_havoc_step: { u32 r = rand_below(afl, MUT_STRATEGY_ARRAY_SIZE), item; diff --git a/include/xxhash.h b/include/xxhash.h index 7bc0a14e..a8bd6f27 100644 --- a/include/xxhash.h +++ b/include/xxhash.h @@ -365,7 +365,7 @@ typedef uint32_t XXH32_hash_t; (defined(__cplusplus) || \ (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) #include -typedef uint32_t XXH32_hash_t; +typedef uint32_t XXH32_hash_t; #else #include @@ -1082,7 +1082,7 @@ struct XXH64_state_s { #include #define XXH_ALIGN(n) alignas(n) #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ - /* In C++ alignas() is a keyword */ + /* In C++ alignas() is a keyword */ #define XXH_ALIGN(n) alignas(n) #elif defined(__GNUC__) #define XXH_ALIGN(n) __attribute__((aligned(n))) @@ -3031,8 +3031,8 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src) { __STDC_VERSION__ >= 199901L /* >= C99 */ #define XXH_RESTRICT restrict #else - /* Note: it might be useful to define __restrict or __restrict__ for - * some C++ compilers */ + /* Note: it might be useful to define __restrict or __restrict__ for + * some C++ compilers */ #define XXH_RESTRICT /* disable */ #endif @@ -3492,8 +3492,8 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) { #define XXH_vec_mulo vec_mulo #define XXH_vec_mule vec_mule #elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) - /* Clang has a better way to control this, we can just use the builtin - * which doesn't swap. */ + /* Clang has a better way to control this, we can just use the builtin + * which doesn't swap. */ #define XXH_vec_mulo __builtin_altivec_vmulouw #define XXH_vec_mule __builtin_altivec_vmuleuw #else @@ -3604,15 +3604,15 @@ XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { #include #define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) #else - /* - * Downcast + upcast is usually better than masking on older compilers - * like GCC 4.2 (especially 32-bit ones), all without affecting newer - * compilers. - * - * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both - * operands and perform a full 64x64 multiply -- entirely redundant on - * 32-bit. - */ + /* + * Downcast + upcast is usually better than masking on older compilers + * like GCC 4.2 (especially 32-bit ones), all without affecting newer + * compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both + * operands and perform a full 64x64 multiply -- entirely redundant on + * 32-bit. + */ #define XXH_mult32to64(x, y) \ ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) #endif diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index bca1f927..9cd1dc59 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -90,7 +90,7 @@ class CmpLogInstructions : public ModulePass { #if LLVM_MAJOR >= 11 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); #else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR >= 4 StringRef getPassName() const override { diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index c3fbed8d..54e9ddf3 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -85,7 +85,7 @@ class CmpLogRoutines : public ModulePass { #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); #else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR >= 4 StringRef getPassName() const override { diff --git a/instrumentation/cmplog-switches-pass.cc b/instrumentation/cmplog-switches-pass.cc index 38de669d..01da6da7 100644 --- a/instrumentation/cmplog-switches-pass.cc +++ b/instrumentation/cmplog-switches-pass.cc @@ -85,7 +85,7 @@ class CmplogSwitches : public ModulePass { #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); #else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR < 4 const char *getPassName() const override { diff --git a/instrumentation/split-switches-pass.so.cc b/instrumentation/split-switches-pass.so.cc index dcd89652..e3dfea0d 100644 --- a/instrumentation/split-switches-pass.so.cc +++ b/instrumentation/split-switches-pass.so.cc @@ -84,7 +84,7 @@ class SplitSwitchesTransform : public ModulePass { #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); #else - bool runOnModule(Module &M) override; + bool runOnModule(Module &M) override; #if LLVM_VERSION_MAJOR >= 4 StringRef getPassName() const override { diff --git a/qemu_mode/libqasan/dlmalloc.c b/qemu_mode/libqasan/dlmalloc.c index b459eb7b..1919ae26 100644 --- a/qemu_mode/libqasan/dlmalloc.c +++ b/qemu_mode/libqasan/dlmalloc.c @@ -771,8 +771,8 @@ MAX_RELEASE_CHECK_RATE default: 4095 unless not HAVE_MMAP #include "/usr/include/malloc.h" #else /* HAVE_USR_INCLUDE_MALLOC_H */ #ifndef STRUCT_MALLINFO_DECLARED - /* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is - * defined */ + /* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is + * defined */ #define _STRUCT_MALLINFO #define STRUCT_MALLINFO_DECLARED 1 struct mallinfo { @@ -1660,10 +1660,10 @@ extern size_t getpagesize(); #define is_aligned(A) (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0) /* the number of bytes to offset an address to align it */ - #define align_offset(A) \ - ((((size_t)(A)&CHUNK_ALIGN_MASK) == 0) \ - ? 0 \ - : ((MALLOC_ALIGNMENT - ((size_t)(A)&CHUNK_ALIGN_MASK)) & \ + #define align_offset(A) \ + ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0) \ + ? 0 \ + : ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & \ CHUNK_ALIGN_MASK)) /* -------------------------- MMAP preliminaries ------------------------- */ @@ -1715,10 +1715,10 @@ static FORCEINLINE int unixmunmap(void *ptr, size_t size) { #define MUNMAP_DEFAULT(a, s) unixmunmap((a), (s)) #else /* MAP_ANONYMOUS */ - /* - Nearly all versions of mmap support MAP_ANONYMOUS, so the following - is unlikely to be needed, but is supplied just in case. - */ + /* + Nearly all versions of mmap support MAP_ANONYMOUS, so the following + is unlikely to be needed, but is supplied just in case. + */ #define MMAP_FLAGS (MAP_PRIVATE) static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */ #define MMAP_DEFAULT(s) \ @@ -1965,7 +1965,7 @@ static FORCEINLINE void x86_clear_lock(int *sl) { #endif /* ... gcc spins locks ... */ - /* How to yield for a spin lock */ + /* How to yield for a spin lock */ #define SPINS_PER_YIELD 63 #if defined(_MSC_VER) #define SLEEP_EX_DURATION 50 /* delay for yield/sleep */ @@ -2008,11 +2008,11 @@ static MLOCK_T malloc_global_mutex = 0; #define CURRENT_THREAD GetCurrentThreadId() #define EQ_OWNER(X, Y) ((X) == (Y)) #else - /* - Note: the following assume that pthread_t is a type that can be - initialized to (casted) zero. If this is not the case, you will need - to somehow redefine these or not use spin locks. - */ + /* + Note: the following assume that pthread_t is a type that can be + initialized to (casted) zero. If this is not the case, you will need + to somehow redefine these or not use spin locks. + */ #define THREAD_ID_T pthread_t #define CURRENT_THREAD pthread_self() #define EQ_OWNER(X, Y) pthread_equal(X, Y) @@ -2169,7 +2169,7 @@ static int pthread_init_lock(MLOCK_T *lk) { #endif /* ... lock types ... */ - /* Common code for all lock types */ + /* Common code for all lock types */ #define USE_LOCK_BIT (2U) #ifndef ACQUIRE_MALLOC_GLOBAL_LOCK @@ -3077,7 +3077,7 @@ static size_t traverse_and_check(mstate m); /* The size of the smallest chunk held in bin with index i */ #define minsize_for_tree_index(i) \ ((SIZE_T_ONE << (((i) >> 1) + TREEBIN_SHIFT)) | \ - (((size_t)((i)&SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1))) + (((size_t)((i) & SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1))) /* ------------------------ Operations on bin maps ----------------------- */ @@ -3245,7 +3245,7 @@ static size_t traverse_and_check(mstate m); #else /* FOOTERS */ - /* Set foot of inuse chunk to be xor of mstate and seed */ + /* Set foot of inuse chunk to be xor of mstate and seed */ #define mark_inuse_foot(M, p, s) \ (((mchunkptr)((char *)(p) + (s)))->prev_foot = \ ((size_t)(M) ^ mparams.magic)) diff --git a/qemu_mode/libqasan/malloc.c b/qemu_mode/libqasan/malloc.c index d2db3856..4448f480 100644 --- a/qemu_mode/libqasan/malloc.c +++ b/qemu_mode/libqasan/malloc.c @@ -80,8 +80,8 @@ static unsigned char __tmp_alloc_zone[TMP_ZONE_SIZE]; #else // From dlmalloc.c -void *dlmalloc(size_t); -void dlfree(void *); +void *dlmalloc(size_t); +void dlfree(void *); #define backend_malloc dlmalloc #define backend_free dlfree diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 2003be1f..b2306996 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -577,13 +577,13 @@ u8 fuzz_one_original(afl_state_t *afl) { * SIMPLE BITFLIP (+dictionary construction) * *********************************************/ -#define FLIP_BIT(_ar, _b) \ - do { \ - \ - u8 *_arf = (u8 *)(_ar); \ - u32 _bf = (_b); \ - _arf[(_bf) >> 3] ^= (128 >> ((_bf)&7)); \ - \ +#define FLIP_BIT(_ar, _b) \ + do { \ + \ + u8 *_arf = (u8 *)(_ar); \ + u32 _bf = (_b); \ + _arf[(_bf) >> 3] ^= (128 >> ((_bf) & 7)); \ + \ } while (0) /* Single walking bit. */ @@ -2216,7 +2216,7 @@ havoc_stage: } - retry_havoc_step : { + retry_havoc_step: { u32 r = rand_below(afl, rand_max), item; @@ -3703,13 +3703,13 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { * SIMPLE BITFLIP (+dictionary construction) * *********************************************/ -#define FLIP_BIT(_ar, _b) \ - do { \ - \ - u8 *_arf = (u8 *)(_ar); \ - u32 _bf = (_b); \ - _arf[(_bf) >> 3] ^= (128 >> ((_bf)&7)); \ - \ +#define FLIP_BIT(_ar, _b) \ + do { \ + \ + u8 *_arf = (u8 *)(_ar); \ + u32 _bf = (_b); \ + _arf[(_bf) >> 3] ^= (128 >> ((_bf) & 7)); \ + \ } while (0) /* Single walking bit. */ diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 43b5c8bd..86e7f1cf 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -1828,7 +1828,7 @@ static void try_to_add_to_dictN(afl_state_t *afl, u128 v, u8 size) { for (k = 0; k < size; ++k) { #else - u32 off = 16 - size; + u32 off = 16 - size; for (k = 16 - size; k < 16; ++k) { #endif diff --git a/utils/libtokencap/libtokencap.so.c b/utils/libtokencap/libtokencap.so.c index b21f3068..f4024799 100644 --- a/utils/libtokencap/libtokencap.so.c +++ b/utils/libtokencap/libtokencap.so.c @@ -55,7 +55,7 @@ #elif defined __HAIKU__ #include #elif defined __sun - /* For map addresses the old struct is enough */ +/* For map addresses the old struct is enough */ #include #include #endif @@ -168,7 +168,7 @@ static void __tokencap_load_mappings(void) { #elif defined __FreeBSD__ || defined __OpenBSD__ || defined __NetBSD__ #if defined __FreeBSD__ - int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, __tokencap_pid}; + int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, __tokencap_pid}; #elif defined __OpenBSD__ int mib[] = {CTL_KERN, KERN_PROC_VMMAP, __tokencap_pid}; #elif defined __NetBSD__ @@ -209,7 +209,7 @@ static void __tokencap_load_mappings(void) { #if defined __FreeBSD__ || defined __NetBSD__ #if defined __FreeBSD__ - size_t size = region->kve_structsize; + size_t size = region->kve_structsize; if (size == 0) break; #elif defined __NetBSD__ -- cgit 1.4.1 From 1fa285079f895b3e0b5b347830ce8a7ab980c691 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 30 Nov 2023 11:52:10 +0100 Subject: nit --- utils/aflpp_driver/aflpp_driver.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'utils') diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index 4e8f466d..1104a81e 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -279,7 +279,9 @@ __attribute__((weak)) int main(int argc, char **argv) { */ - if (argc < 2 || strncmp(argv[1], "-h", 2) == 0) + if (argc < 2 || strncmp(argv[1], "-h", 2) == 0 || + strcmp(argv[1], "--help") == 0) { + printf( "============================== INFO ================================\n" "This binary is built for afl++.\n" @@ -296,6 +298,13 @@ __attribute__((weak)) int main(int argc, char **argv) { "option\n" "===================================================================\n", argv[0], argv[0]); + if (strncmp(argv[1], "-h", 2) == 0 || strcmp(argv[1], "--help") == 0) { + + exit(0); + + } + + } return LLVMFuzzerRunDriver(&argc, &argv, LLVMFuzzerTestOneInput); -- cgit 1.4.1 From d02036adfd098766ce9576905613cb7911e315d5 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 1 Dec 2023 07:20:00 +0100 Subject: fix --- instrumentation/afl-compiler-rt.o.c | 2 +- utils/aflpp_driver/aflpp_driver.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'utils') diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 106892e2..def59b6b 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -183,7 +183,7 @@ static u8 _is_sancov; /* Debug? */ -static u32 __afl_debug; +/*static*/ u32 __afl_debug; /* Already initialized markers */ diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index 1104a81e..3f8e1ef7 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -298,7 +298,8 @@ __attribute__((weak)) int main(int argc, char **argv) { "option\n" "===================================================================\n", argv[0], argv[0]); - if (strncmp(argv[1], "-h", 2) == 0 || strcmp(argv[1], "--help") == 0) { + if (argc == 2 && strncmp(argv[1], "-h", 2) == 0 || + strcmp(argv[1], "--help") == 0) { exit(0); -- cgit 1.4.1 From 858e0bfd05894d07630d8a56bb25d56a8206b2b7 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 1 Dec 2023 07:21:43 +0100 Subject: fix --- utils/aflpp_driver/aflpp_driver.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'utils') diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index 3f8e1ef7..dab7fd95 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -298,8 +298,8 @@ __attribute__((weak)) int main(int argc, char **argv) { "option\n" "===================================================================\n", argv[0], argv[0]); - if (argc == 2 && strncmp(argv[1], "-h", 2) == 0 || - strcmp(argv[1], "--help") == 0) { + if (argc == 2 && + (strncmp(argv[1], "-h", 2) == 0 || strcmp(argv[1], "--help") == 0)) { exit(0); -- cgit 1.4.1 From ae9cdb34e4fdc10c7c2d1c775238a7501fda288a Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 14 Dec 2023 16:04:00 +0100 Subject: AFL_FUZZER_LOOPCOUNT --- docs/Changelog.md | 1 + utils/aflpp_driver/aflpp_driver.c | 7 +++++++ 2 files changed, 8 insertions(+) (limited to 'utils') diff --git a/docs/Changelog.md b/docs/Changelog.md index b2e9fbf6..7faa0ab3 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -25,6 +25,7 @@ - fix for a few string compare transform functions for LAF - frida_mode: - fixes support for large map offsets + - support for AFL_FUZZER_LOOPCOUNT for afl.rs and LLVMFuzzerTestOneInput - afl-cmin/afl-cmin.bash: prevent unneeded file errors - added new tool afl-addseeds that adds new seeds to a running campaign - added benchmark/benchmark.py if you want to see how good your fuzzing diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index dab7fd95..9ffb2383 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -292,6 +292,7 @@ __attribute__((weak)) int main(int argc, char **argv) { "afl-fuzz will run N iterations before re-spawning the process " "(default: " "INT_MAX)\n" + "You can also use AFL_FUZZER_LOOPCOUNT to set N\n" "For stdin input processing, pass '-' as single command line option.\n" "For file input processing, pass '@@' as single command line option.\n" "To use with afl-cmin or afl-cmin.bash pass '-' as single command line " @@ -379,6 +380,12 @@ __attribute__((weak)) int LLVMFuzzerRunDriver( } + if (getenv("AFL_FUZZER_LOOPCOUNT")) { + + N = atoi(getenv("AFL_FUZZER_LOOPCOUNT")); + + } + assert(N > 0); __afl_manual_init(); -- cgit 1.4.1 From 8fedf4998449d5b6b909a1118fc2e152e4d2e6e7 Mon Sep 17 00:00:00 2001 From: Davide Quarta Date: Tue, 23 Jan 2024 19:36:49 +0100 Subject: replay mode support --- .gitignore | 1 + include/afl-fuzz.h | 4 + include/config.h | 5 +- include/persistent_replay.h | 149 ++++++++++++++++++++++++++++ instrumentation/afl-compiler-rt.o.c | 36 +++++++ src/afl-forkserver.c | 79 +++++++++------ src/afl-fuzz-init.c | 6 ++ src/afl-fuzz.c | 2 +- utils/persistent_mode/Makefile | 3 +- utils/persistent_mode/persistent_demo_new.c | 15 +-- 10 files changed, 257 insertions(+), 43 deletions(-) create mode 100644 include/persistent_replay.h (limited to 'utils') diff --git a/.gitignore b/.gitignore index f76a86fc..891ced9f 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ utils/optimin/build utils/optimin/optimin utils/persistent_mode/persistent_demo utils/persistent_mode/persistent_demo_new +utils/persistent_mode/persistent_demo_new_compat utils/persistent_mode/test-instr utils/plot_ui/afl-plot-ui vuln_prog diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index f1813df6..864bc6b6 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -124,6 +124,10 @@ #define CASE_PREFIX "id_" #endif /* ^!SIMPLE_FILES */ +#ifdef AFL_PERSISTENT_RECORD + #define RECORD_PREFIX "RECORD:" +#endif + #define STAGE_BUF_SIZE (64) /* usable size for stage name buf in afl_state */ // Little helper to access the ptr to afl->##name_buf - for use in afl_realloc. diff --git a/include/config.h b/include/config.h index 63340650..1649f110 100644 --- a/include/config.h +++ b/include/config.h @@ -83,7 +83,10 @@ will be kept and written to the crash/ directory as RECORD:... files. Note that every crash will be written, not only unique ones! */ -// #define AFL_PERSISTENT_RECORD +// #define AFL_PERSISTENT_RECORD + +/* Builds compiler-rt with support to replay persistent records */ +// #define AFL_PERSISTENT_REPLAY /* console output colors: There are three ways to configure its behavior * 1. default: colored outputs fixed on: defined USE_COLOR && defined diff --git a/include/persistent_replay.h b/include/persistent_replay.h new file mode 100644 index 00000000..b1a55e9f --- /dev/null +++ b/include/persistent_replay.h @@ -0,0 +1,149 @@ +#ifndef _HAVE_PERSISTENT_REPLAY_H +#define _HAVE_PERSISTENT_REPLAY_H + +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned short int is_replay_record; +static unsigned int replay_record; +static unsigned int replay_record_cnt; +static char replay_record_path[PATH_MAX]; +static char **record_arg; +static char *replay_record_dir; +static struct dirent **record_list; + +static int select_files(const struct dirent *dirbuf) { + + char fn[4096]; + + if (dirbuf->d_name[0] == '.'){ + return 0; + } else { + snprintf(fn, sizeof(fn), "RECORD:%06u", replay_record); + return !!strstr(dirbuf->d_name, fn); + } +} + +static int compare_files(const struct dirent **da, const struct dirent **db) { + + unsigned int c1=0, c2=0; + + sscanf((*da)->d_name, "RECORD:%*u,cnt:%06u", &c1); + sscanf((*db)->d_name, "RECORD:%*u,cnt:%06u", &c2); + + return c1-c2; +} + +__attribute__((destructor)) static void __afl_record_replay_destroy(void){ + for (int i=0; i < replay_record_cnt; i++) { + free(record_list[i]); + } + free(record_list); +} + +__attribute__((constructor)) static void __afl_record_replay_init(int argc, char **argv) { + + char **argp; + + /* caveat: if harness uses @@ and we don't pass it, it will regardless loop the number of iterations defined for AFL_LOOP (on the same file)*/ + if(!(is_replay_record = !!getenv("AFL_PERSISTENT_REPLAY"))){ + // printf("[warning] AFL_PERSISTENT_REPLAY not set.\n"); + return; + } + + replay_record = atoi(getenv("AFL_PERSISTENT_REPLAY")); + replay_record_dir = getenv("AFL_PERSISTENT_DIR"); + replay_record_cnt = scandir(replay_record_dir ? replay_record_dir : "./", &record_list, select_files, compare_files); + + if (!replay_record_cnt){ + printf("[error] Can't find the requested record!\n"); + is_replay_record = 0; + } + + argp = argv; + while (*argp){ + if (!strcmp(*argp, "@@")){ + record_arg = argp; + *record_arg = replay_record_path; + break; + } + ++argp; + } + +} + +/* only used if explictly included for compatibility + compiling without afl-cc */ + +#ifdef AFL_COMPAT + +#ifndef PATH_MAX + #define PATH_MAX 4096 +#endif + +#define FUZZ_BUF_SIZE 1024000 + +// extern ssize_t read(int fildes, void *buf, size_t nbyte); + +//extern int __afl_persistent_loop(unsigned int max_cnt); +//extern unsigned char fuzz_buf[]; + +#ifndef __AFL_HAVE_MANUAL_CONTROL + #define __AFL_HAVE_MANUAL_CONTROL +#endif + +#define __AFL_FUZZ_TESTCASE_LEN (read(0, fuzz_buf, FUZZ_BUF_SIZE)) +#define __AFL_FUZZ_TESTCASE_BUF fuzz_buf +#define __AFL_FUZZ_INIT() void sync(void); +#define __AFL_INIT() sync() +#define __AFL_LOOP(x) __afl_persistent_loop(x) + +unsigned char fuzz_buf[FUZZ_BUF_SIZE]; + +int __afl_persistent_loop(unsigned int max_cnt) { + + static unsigned int cycle_cnt = 1; + static unsigned short int inited = 0; + char tcase[PATH_MAX]; + + if( is_replay_record ){ + + if (!inited){ + cycle_cnt = replay_record_cnt; + inited = 1; + } + + snprintf(tcase, PATH_MAX, "%s/%s", + replay_record_dir ? replay_record_dir : "./", + record_list[replay_record_cnt-cycle_cnt]->d_name); + + + if (record_arg) { + *record_arg = tcase; + } else { + int fd = open(tcase, O_RDONLY); + dup2(fd, 0); + close(fd); + } + + } else { + + if (!inited){ + cycle_cnt = max_cnt; + inited = 1; + } + + } + + return cycle_cnt--; +} + +#endif // AFL_COMPAT + +#endif // _HAVE_PERSISTENT_REPLAY_H \ No newline at end of file diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 39a762b6..0fa22aee 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -83,6 +83,10 @@ #include #include +#ifdef AFL_PERSISTENT_REPLAY +#include "persistent_replay.h" +#endif + /* Globals needed by the injected instrumentation. The __afl_area_initial region is used for instrumentation output before __afl_map_shm() has a chance to run. It will end up as .comm, so it shouldn't be too wasteful. */ @@ -1338,6 +1342,38 @@ int __afl_persistent_loop(unsigned int max_cnt) { static u8 first_pass = 1; static u32 cycle_cnt; +#ifdef AFL_PERSISTENT_REPLAY + +#ifndef PATH_MAX + #define PATH_MAX 4096 +#endif + + static u8 inited = 0; + char tcase[PATH_MAX]; + + if( unlikely(is_replay_record) ){ + + if (!inited){ + cycle_cnt = replay_record_cnt; + inited = 1; + } + + snprintf(tcase, PATH_MAX, "%s/%s", + replay_record_dir ? replay_record_dir : "./", + record_list[replay_record_cnt-cycle_cnt]->d_name); + + if (record_arg) { + *record_arg = tcase; + } else { + int fd = open(tcase, O_RDONLY); + dup2(fd, 0); + close(fd); + } + return cycle_cnt--; + } else + +#endif + if (first_pass) { /* Make sure that every iteration of __AFL_LOOP() starts with a clean slate. diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 3f9bfa72..f8dd783f 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -1591,6 +1591,11 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, u32 exec_ms; u32 write_value = fsrv->last_run_timed_out; +#ifdef AFL_PERSISTENT_RECORD + fsrv_run_result_t retval = FSRV_RUN_OK; + char *persistent_out_fmt; +#endif + #ifdef __linux__ if (fsrv->nyx_mode) { @@ -1684,7 +1689,7 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, } -#ifdef AFL_PERSISTENT_RECORD +#ifdef AFL_eERSISTENT_RECORD // end of persistent loop? if (unlikely(fsrv->persistent_record && fsrv->persistent_record_pid != fsrv->child_pid)) { @@ -1790,8 +1795,14 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, if (unlikely(fsrv->last_run_timed_out)) { fsrv->last_kill_signal = fsrv->child_kill_signal; - return FSRV_RUN_TMOUT; +#ifndef AFL_PERSISTENT_RECORD + return FSRV_RUN_TMOUT; +#else + retval = FSRV_RUN_TMOUT; + persistent_out_fmt = "%s/hangs/RECORD:%06u,cnt:%06u"; + goto store_persistent_record; +#endif } /* Did we crash? @@ -1811,48 +1822,58 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, (fsrv->uses_crash_exitcode && WEXITSTATUS(fsrv->child_status) == fsrv->crash_exitcode))) { -#ifdef AFL_PERSISTENT_RECORD - if (unlikely(fsrv->persistent_record)) { + /* For a proper crash, set last_kill_signal to WTERMSIG, else set it to 0 */ + fsrv->last_kill_signal = + WIFSIGNALED(fsrv->child_status) ? WTERMSIG(fsrv->child_status) : 0; - char fn[PATH_MAX]; - u32 i, writecnt = 0; - for (i = 0; i < fsrv->persistent_record; ++i) { +#ifndef AFL_PERSISTENT_RECORD + return FSRV_RUN_CRASH; +#else + retval = FSRV_RUN_CRASH; + persistent_out_fmt = "%s/crashes/RECORD:%06u,cnt:%06u"; + goto store_persistent_record; +#endif - u32 entry = (i + fsrv->persistent_record_idx) % fsrv->persistent_record; - u8 *data = fsrv->persistent_record_data[entry]; - u32 len = fsrv->persistent_record_len[entry]; - if (likely(len && data)) { + } - snprintf(fn, sizeof(fn), "%s/RECORD:%06u,cnt:%06u", - fsrv->persistent_record_dir, fsrv->persistent_record_cnt, - writecnt++); - int fd = open(fn, O_CREAT | O_TRUNC | O_WRONLY, 0644); - if (fd >= 0) { + /* success :) */ + return FSRV_RUN_OK; + +#ifdef AFL_PERSISTENT_RECORD +store_persistent_record: + if (unlikely(retval == FSRV_RUN_CRASH || retval == FSRV_RUN_TMOUT) && + unlikely(fsrv->persistent_record)) { - ck_write(fd, data, len, fn); - close(fd); + char fn[PATH_MAX]; + u32 i, writecnt = 0; + for (i = 0; i < fsrv->persistent_record; ++i) { - } + u32 entry = (i + fsrv->persistent_record_idx) % fsrv->persistent_record; + u8 *data = fsrv->persistent_record_data[entry]; + u32 len = fsrv->persistent_record_len[entry]; + if (likely(len && data)) { + + snprintf(fn, sizeof(fn), persistent_out_fmt, + fsrv->persistent_record_dir, fsrv->persistent_record_cnt, + writecnt++); + int fd = open(fn, O_CREAT | O_TRUNC | O_WRONLY, 0644); + if (fd >= 0) { + + ck_write(fd, data, len, fn); + close(fd); } } - ++fsrv->persistent_record_cnt; - } -#endif - - /* For a proper crash, set last_kill_signal to WTERMSIG, else set it to 0 */ - fsrv->last_kill_signal = - WIFSIGNALED(fsrv->child_status) ? WTERMSIG(fsrv->child_status) : 0; - return FSRV_RUN_CRASH; + ++fsrv->persistent_record_cnt; } - /* success :) */ - return FSRV_RUN_OK; + return retval; +#endif } diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 35932913..5b7dc4c1 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -1915,6 +1915,9 @@ static void handle_existing_out_dir(afl_state_t *afl) { } +#ifdef AFL_PERSISTENT_RECORD + delete_files(fn, RECORD_PREFIX); +#endif if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } ck_free(fn); @@ -1947,6 +1950,9 @@ static void handle_existing_out_dir(afl_state_t *afl) { } +#ifdef AFL_PERSISTENT_RECORD + delete_files(fn, RECORD_PREFIX); +#endif if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; } ck_free(fn); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 17949fd7..40c30472 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2163,7 +2163,7 @@ int main(int argc, char **argv_orig, char **envp) { } - afl->fsrv.persistent_record_dir = alloc_printf("%s/crashes", afl->out_dir); + afl->fsrv.persistent_record_dir = alloc_printf("%s", afl->out_dir); } diff --git a/utils/persistent_mode/Makefile b/utils/persistent_mode/Makefile index e348c46c..64de82a7 100644 --- a/utils/persistent_mode/Makefile +++ b/utils/persistent_mode/Makefile @@ -1,10 +1,11 @@ all: ../../afl-clang-fast -o persistent_demo persistent_demo.c ../../afl-clang-fast -o persistent_demo_new persistent_demo_new.c + gcc -g -I ../../include -o persistent_demo_new_compat persistent_demo_new.c AFL_DONT_OPTIMIZE=1 ../../afl-clang-fast -o test-instr test-instr.c document: AFL_DONT_OPTIMIZE=1 ../../afl-clang-fast -D_AFL_DOCUMENT_MUTATIONS -o test-instr test-instr.c clean: - rm -f persistent_demo persistent_demo_new test-instr + rm -f persistent_demo persistent_demo_new persistent_demo_new_compat test-instr diff --git a/utils/persistent_mode/persistent_demo_new.c b/utils/persistent_mode/persistent_demo_new.c index 285f50aa..40ada9e1 100644 --- a/utils/persistent_mode/persistent_demo_new.c +++ b/utils/persistent_mode/persistent_demo_new.c @@ -31,17 +31,8 @@ /* this lets the source compile without afl-clang-fast/lto */ #ifndef __AFL_FUZZ_TESTCASE_LEN - -ssize_t fuzz_len; -unsigned char fuzz_buf[1024000]; - - #define __AFL_FUZZ_TESTCASE_LEN fuzz_len - #define __AFL_FUZZ_TESTCASE_BUF fuzz_buf - #define __AFL_FUZZ_INIT() void sync(void); - #define __AFL_LOOP(x) \ - ((fuzz_len = read(0, fuzz_buf, sizeof(fuzz_buf))) > 0 ? 1 : 0) - #define __AFL_INIT() sync() - +#define AFL_COMPAT +#include "persistent_replay.h" #endif __AFL_FUZZ_INIT(); @@ -95,6 +86,8 @@ int main(int argc, char **argv) { if (buf[5] == '!') { printf("six\n"); + char *nullo = NULL+1; + *nullo = 'p'; abort(); } -- cgit 1.4.1 From 58b80b68bc5538bad2cf4c858229111f58282424 Mon Sep 17 00:00:00 2001 From: "Christian Holler (:decoder)" Date: Fri, 26 Jan 2024 15:46:56 +0100 Subject: Dynamic instrumentation filtering for LLVM native (#1971) * Add two dynamic instrumentation filter methods to runtime * Always use pc-table with native pcguard * Add make_symbol_list.py and README --- instrumentation/afl-compiler-rt.o.c | 279 +++++++++++++++++++++++++--- src/afl-cc.c | 26 +-- utils/dynamic_covfilter/README.md | 55 ++++++ utils/dynamic_covfilter/make_symbol_list.py | 73 ++++++++ 4 files changed, 393 insertions(+), 40 deletions(-) create mode 100644 utils/dynamic_covfilter/README.md create mode 100644 utils/dynamic_covfilter/make_symbol_list.py (limited to 'utils') diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 39a762b6..8e55d6a0 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -22,6 +22,10 @@ #define __USE_GNU #endif #include + +__attribute__((weak)) void __sanitizer_symbolize_pc(void *, const char *fmt, + char *out_buf, + size_t out_buf_size); #endif #ifdef __ANDROID__ @@ -124,8 +128,8 @@ struct afl_module_info_t { uintptr_t base_address; // PC Guard start/stop - u32 start; - u32 stop; + u32 *start; + u32 *stop; // PC Table begin/end const uintptr_t *pcs_beg; @@ -147,6 +151,18 @@ afl_module_info_t *__afl_module_info = NULL; u32 __afl_pcmap_size = 0; uintptr_t *__afl_pcmap_ptr = NULL; + +typedef struct { + + uintptr_t start; + u32 len; + +} FilterPCEntry; + +u32 __afl_filter_pcs_size = 0; +FilterPCEntry *__afl_filter_pcs = NULL; +u8 *__afl_filter_pcs_module = NULL; + #endif // __AFL_CODE_COVERAGE /* 1 if we are running in afl, and the forkserver was started, else 0 */ @@ -1587,15 +1603,116 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *guard) { } #ifdef __AFL_CODE_COVERAGE -void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, - const uintptr_t *pcs_end) { +void afl_read_pc_filter_file(const char *filter_file) { - if (__afl_debug) { + FILE *file; + char ch; + + file = fopen(filter_file, "r"); + if (file == NULL) { + + perror("Error opening file"); + return; + + } + + // Check how many PCs we expect to read + while ((ch = fgetc(file)) != EOF) { + + if (ch == '\n') { __afl_filter_pcs_size++; } + + } + + // Rewind to actually read the PCs + fseek(file, 0, SEEK_SET); + + __afl_filter_pcs = malloc(__afl_filter_pcs_size * sizeof(FilterPCEntry)); + if (!__afl_filter_pcs) { + + perror("Error allocating PC array"); + return; + + } + + for (size_t i = 0; i < __afl_filter_pcs_size; i++) { + + fscanf(file, "%lx", &(__afl_filter_pcs[i].start)); + ch = fgetc(file); // Read tab + fscanf(file, "%u", &(__afl_filter_pcs[i].len)); + ch = fgetc(file); // Read tab + + if (!__afl_filter_pcs_module) { + + // Read the module name and store it. + // TODO: We only support one module here right now although + // there is technically no reason to support multiple modules + // in one go. + size_t max_module_len = 255; + size_t i = 0; + __afl_filter_pcs_module = malloc(max_module_len); + while (i < max_module_len - 1 && + (__afl_filter_pcs_module[i] = fgetc(file)) != '\t') { + + ++i; + + } - fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init called\n"); + __afl_filter_pcs_module[i] = '\0'; + fprintf(stderr, "DEBUGXXX: Read module name %s\n", + __afl_filter_pcs_module); + + } + + while ((ch = fgetc(file)) != '\n' && ch != EOF) + ; + + } + + fclose(file); + +} + +u32 locate_in_pcs(uintptr_t needle, u32 *index) { + + size_t lower_bound = 0; + size_t upper_bound = __afl_filter_pcs_size - 1; + + while (lower_bound < __afl_filter_pcs_size && lower_bound <= upper_bound) { + + size_t current_index = lower_bound + (upper_bound - lower_bound) / 2; + + if (__afl_filter_pcs[current_index].start <= needle) { + + if (__afl_filter_pcs[current_index].start + + __afl_filter_pcs[current_index].len > + needle) { + + // Hit + *index = current_index; + return 1; + + } else { + + lower_bound = current_index + 1; + + } + + } else { + + if (!current_index) { break; } + upper_bound = current_index - 1; + + } } + return 0; + +} + +void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + const uintptr_t *pcs_end) { + // If for whatever reason, we cannot get dlinfo here, then pc_guard_init also // couldn't get it and we'd end up attributing to the wrong module. Dl_info dlinfo; @@ -1608,6 +1725,16 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, } + if (__afl_debug) { + + fprintf( + stderr, + "DEBUG: (%u) __sanitizer_cov_pcs_init called for module %s with %ld " + "PCs\n", + getpid(), dlinfo.dli_fname, pcs_end - pcs_beg); + + } + afl_module_info_t *last_module_info = __afl_module_info; while (last_module_info && last_module_info->next) { @@ -1623,34 +1750,78 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, } + if (strcmp(dlinfo.dli_fname, last_module_info->name)) { + + // This can happen with modules being loaded after the forkserver + // where we decide to not track the module. In that case we must + // not track it here either. + fprintf( + stderr, + "WARNING: __sanitizer_cov_pcs_init module info mismatch: %s vs %s\n", + dlinfo.dli_fname, last_module_info->name); + return; + + } + last_module_info->pcs_beg = pcs_beg; last_module_info->pcs_end = pcs_end; + // This is a direct filter based on symbolizing inside the runtime. + // It should only be used with smaller binaries to avoid long startup + // times. Currently, this only supports a single token to scan for. + const char *pc_filter = getenv("AFL_PC_FILTER"); + + // This is a much faster PC filter based on pre-symbolized input data + // that is sorted for fast lookup through binary search. This method + // of filtering is suitable even for very large binaries. + const char *pc_filter_file = getenv("AFL_PC_FILTER_FILE"); + if (pc_filter_file && !__afl_filter_pcs) { + + afl_read_pc_filter_file(pc_filter_file); + + } + // Now update the pcmap. If this is the last module coming in, after all // pre-loaded code, then this will also map all of our delayed previous // modules. - - if (!__afl_pcmap_ptr) { return; } - + // for (afl_module_info_t *mod_info = __afl_module_info; mod_info; mod_info = mod_info->next) { if (mod_info->mapped) { continue; } + if (!mod_info->start) { + + fprintf(stderr, + "ERROR: __sanitizer_cov_pcs_init called with mod_info->start == " + "NULL (%s)\n", + mod_info->name); + abort(); + + } + PCTableEntry *start = (PCTableEntry *)(mod_info->pcs_beg); PCTableEntry *end = (PCTableEntry *)(mod_info->pcs_end); + if (!*mod_info->stop) { continue; } + u32 in_module_index = 0; while (start < end) { - if (mod_info->start + in_module_index >= __afl_map_size) { + if (*mod_info->start + in_module_index >= __afl_map_size) { - fprintf(stderr, "ERROR: __sanitizer_cov_pcs_init out of bounds?!\n"); + fprintf(stderr, + "ERROR: __sanitizer_cov_pcs_init out of bounds?! Start: %u " + "Stop: %u Map Size: %u (%s)\n", + *mod_info->start, *mod_info->stop, __afl_map_size, + mod_info->name); abort(); } + u32 orig_start_index = *mod_info->start; + uintptr_t PC = start->PC; // This is what `GetPreviousInstructionPc` in sanitizer runtime does @@ -1660,7 +1831,58 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, // Calculate relative offset in module PC = PC - mod_info->base_address; - __afl_pcmap_ptr[mod_info->start + in_module_index] = PC; + if (__afl_pcmap_ptr) { + + __afl_pcmap_ptr[orig_start_index + in_module_index] = PC; + + } + + if (pc_filter) { + + char PcDescr[1024]; + // This function is a part of the sanitizer run-time. + // To use it, link with AddressSanitizer or other sanitizer. + __sanitizer_symbolize_pc((void *)start->PC, "%p %F %L", PcDescr, + sizeof(PcDescr)); + + if (strstr(PcDescr, pc_filter)) { + + if (__afl_debug) + fprintf( + stderr, + "DEBUG: Selective instrumentation match: %s (PC %p Index %u)\n", + PcDescr, (void *)start->PC, + *(mod_info->start + in_module_index)); + // No change to guard needed + + } else { + + // Null out the guard to disable this edge + *(mod_info->start + in_module_index) = 0; + + } + + } + + if (__afl_filter_pcs && strstr(mod_info->name, __afl_filter_pcs_module)) { + + u32 result_index; + if (locate_in_pcs(PC, &result_index)) { + + if (__afl_debug) + fprintf(stderr, + "DEBUG: Selective instrumentation match: (PC %lx File " + "Index %u PC Index %u)\n", + PC, result_index, in_module_index); + + } else { + + // Null out the guard to disable this edge + *(mod_info->start + in_module_index) = 0; + + } + + } start++; in_module_index++; @@ -1671,8 +1893,10 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, if (__afl_debug) { - fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init initialized %u PCs\n", - in_module_index); + fprintf(stderr, + "DEBUG: __sanitizer_cov_pcs_init successfully mapped %s with %u " + "PCs\n", + mod_info->name, in_module_index); } @@ -1706,9 +1930,9 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { fprintf( stderr, "DEBUG: Running __sanitizer_cov_trace_pc_guard_init: %p-%p (%lu edges) " - "after_fs=%u\n", + "after_fs=%u *start=%u\n", start, stop, (unsigned long)(stop - start), - __afl_already_initialized_forkserver); + __afl_already_initialized_forkserver, *start); } @@ -1740,8 +1964,8 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { mod_info->id = last_module_info ? last_module_info->id + 1 : 0; mod_info->name = strdup(dlinfo.dli_fname); mod_info->base_address = (uintptr_t)dlinfo.dli_fbase; - mod_info->start = 0; - mod_info->stop = 0; + mod_info->start = NULL; + mod_info->stop = NULL; mod_info->pcs_beg = NULL; mod_info->pcs_end = NULL; mod_info->mapped = 0; @@ -1757,8 +1981,12 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { } - fprintf(stderr, "[pcmap] Module: %s Base Address: %p\n", dlinfo.dli_fname, - dlinfo.dli_fbase); + if (__afl_debug) { + + fprintf(stderr, "[pcmap] Module: %s Base Address: %p\n", + dlinfo.dli_fname, dlinfo.dli_fbase); + + } } @@ -1861,12 +2089,17 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { #ifdef __AFL_CODE_COVERAGE if (mod_info) { - mod_info->start = *orig_start; - mod_info->stop = *(stop - 1); + if (!mod_info->start) { + + mod_info->start = orig_start; + mod_info->stop = stop - 1; + + } + if (__afl_debug) { fprintf(stderr, "DEBUG: [pcmap] Start Index: %u Stop Index: %u\n", - mod_info->start, mod_info->stop); + *(mod_info->start), *(mod_info->stop)); } diff --git a/src/afl-cc.c b/src/afl-cc.c index 6d8e1024..73487188 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -1920,35 +1920,27 @@ void add_native_pcguard(aflcc_state_t *aflcc) { /* If llvm-config doesn't figure out LLVM_MAJOR, just go on anyway and let compiler complain if doesn't work. */ - if (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { - #if LLVM_MAJOR > 0 && LLVM_MAJOR < 6 - FATAL("pcguard instrumentation with pc-table requires LLVM 6.0.1+"); + FATAL("pcguard instrumentation with pc-table requires LLVM 6.0.1+"); #else #if LLVM_MAJOR == 0 - WARNF( - "pcguard instrumentation with pc-table requires LLVM 6.0.1+" - " otherwise the compiler will fail"); + WARNF( + "pcguard instrumentation with pc-table requires LLVM 6.0.1+" + " otherwise the compiler will fail"); #endif + if (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { + insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard,bb,no-prune,pc-table"); -#endif } else { -#if LLVM_MAJOR > 0 && LLVM_MAJOR < 4 - FATAL("pcguard instrumentation requires LLVM 4.0.1+"); -#else - #if LLVM_MAJOR == 0 - WARNF( - "pcguard instrumentation requires LLVM 4.0.1+" - " otherwise the compiler will fail"); - #endif - insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard"); -#endif + insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard,pc-table"); } +#endif + } void add_optimized_pcguard(aflcc_state_t *aflcc) { diff --git a/utils/dynamic_covfilter/README.md b/utils/dynamic_covfilter/README.md new file mode 100644 index 00000000..a64836f1 --- /dev/null +++ b/utils/dynamic_covfilter/README.md @@ -0,0 +1,55 @@ +# Dynamic Instrumentation Filter + +Sometimes it can be beneficial to limit the instrumentation feedback to +specific code locations. It is possible to do so at compile-time by simply +not instrumenting any undesired locations. However, there are situations +where doing this dynamically without requiring a new build can be beneficial. +Especially when dealing with larger builds, it is much more convenient to +select the target code locations at runtime instead of doing so at build time. + +There are two ways of doing this in AFL++: + +## Simple Selection with `AFL_PC_FILTER` + +This approach requires a build with `AFL_INSTRUMENTATION=llvmnative` or +`llvmcodecov` as well as an AddressSanitizer build with debug information. + +By setting the environment variable `AFL_PC_FILTER` to a string, the runtime +symbolizer is enabled in the AFL++ runtime. At startup, the runtime will call +the `__sanitizer_symbolize_pc` API to resolve every PC in every loaded module. +The runtime then matches the result using `strstr` and disables the PC guard +if the symbolized PC does not contain the specified string. + +This approach has the benefit of being very easy to use. The downside is that +it causes significant startup delays with large binaries and that it requires +an AddressSanitizer build. + +This method has no additional runtime overhead after startup. + +## Selection using pre-symbolized data file with `AFL_PC_FILTER_FILE` + +To avoid large startup time delays, a specific module can be pre-symbolized +using the `make_symbol_list.py` script. This script outputs a sorted list of +functions with their respective relative offsets and lengths in the target +binary: + +`python3 make_symbol_list.py libxul.so > libxul.symbols.txt` + +The resulting list can be filtered, e.g. using grep: + +`grep -i "webgl" libxul.symbols.txt > libxul.webgl.symbols.txt` + +Finally, you can run with `AFL_PC_FILTER_FILE=libxul.webgl.symbols.txt` to +restrict instrumentation feedback to the given locations. This approach only +has a minimal startup time delay due to the implementation only using binary +search on the given file per PC rather than reading debug information for every +PC. It also works well with Nyx, where symbolizing is usually disabled for the +target process to avoid delays with frequent crashes. + +Similar to the previous method, This approach requires a build with +`AFL_INSTRUMENTATION=llvmnative` or `llvmcodecov` as well debug information. +However, it does not require the ASan runtime as it doesn't do the symbolizing +in process. Due to the way it maps PCs to symbols, it is less accurate when it +comes to includes and inlines (it assumes all PCs within a function belong to +that function and originate from the same file). For most purposes, this should +be a reasonable simplification to quickly process even the largest binaries. diff --git a/utils/dynamic_covfilter/make_symbol_list.py b/utils/dynamic_covfilter/make_symbol_list.py new file mode 100644 index 00000000..d1dd6ab3 --- /dev/null +++ b/utils/dynamic_covfilter/make_symbol_list.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Written by Christian Holler + +import json +import os +import sys +import subprocess + +if len(sys.argv) != 2: + print("Usage: %s binfile" % os.path.basename(sys.argv[0])) + sys.exit(1) + +binfile = sys.argv[1] + +addr2len = {} +addrs = [] + +output = subprocess.check_output(["objdump", "-t", binfile]).decode("utf-8") +for line in output.splitlines(): + line = line.replace("\t", " ") + components = [x for x in line.split(" ") if x] + if not components: + continue + try: + start_addr = int(components[0], 16) + except ValueError: + continue + + # Length has variable position in objdump output + length = None + for comp in components[1:]: + if len(comp) == 16: + try: + length = int(comp, 16) + break + except: + continue + + if length is None: + print("ERROR: Couldn't determine function section length: %s" % line) + + func = components[-1] + + addrs.append(start_addr) + addr2len[str(hex(start_addr))] = str(length) + +# The search implementation in the AFL runtime expects everything sorted. +addrs.sort() +addrs = [str(hex(addr)) for addr in addrs] + +# We symbolize in one go to speed things up with large binaries. +output = subprocess.check_output([ + "llvm-addr2line", + "--output-style=JSON", + "-f", "-C", "-a", "-e", + binfile], + input="\n".join(addrs).encode("utf-8")).decode("utf-8") + +output = output.strip().splitlines() +for line in output: + output = json.loads(line) + if "Symbol" in output and output["Address"] in addr2len: + final_output = [ + output["Address"], + addr2len[output["Address"]], + os.path.basename(output["ModuleName"]), + output["Symbol"][0]["FileName"], + output["Symbol"][0]["FunctionName"] + ] + print("\t".join(final_output)) -- cgit 1.4.1 From 644e0694509d4019e6f5075c4b900d412f29df32 Mon Sep 17 00:00:00 2001 From: "Christian Holler (:decoder)" Date: Tue, 30 Jan 2024 15:30:18 +0100 Subject: Fixes to afl-cc and documentation (#1974) * Always compile with -ldl when building for CODE_COVERAGE When building with CODE_COVERAGE, the afl runtime contains code that calls `dladdr` which requires -ldl. Under most circumstances, clang already adds this (e.g. when building with pc-table), but there are some circumstances where it isn't added automatically. * Add visibility declaration to __afl_connected When building with hidden visibility, the use of __AFL_LOOP inside such code can cause linker errors due to __afl_connected being declared "hidden". * Update docs to clarify that CODE_COVERAGE=1 is required for dynamic_covfilter --- GNUmakefile | 4 ++++ src/afl-cc.c | 7 ++++++- utils/dynamic_covfilter/README.md | 7 ++++++- 3 files changed, 16 insertions(+), 2 deletions(-) (limited to 'utils') diff --git a/GNUmakefile b/GNUmakefile index b67f9c15..be5b8146 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -66,6 +66,10 @@ ifdef MSAN_BUILD override LDFLAGS += -fsanitize=memory endif +ifdef CODE_COVERAGE + override CFLAGS += -D__AFL_CODE_COVERAGE=1 +endif + ifeq "$(findstring android, $(shell $(CC) --version 2>/dev/null))" "" ifeq "$(shell echo 'int main() {return 0; }' | $(CC) $(CFLAGS) -Werror -x c - -flto=full -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" CFLAGS_FLTO ?= -flto=full diff --git a/src/afl-cc.c b/src/afl-cc.c index 73487188..d11419b0 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -1521,7 +1521,7 @@ void add_defs_persistent_mode(aflcc_state_t *aflcc) { "({ static volatile const char *_B __attribute__((used,unused)); " " _B = (const char*)\"" PERSIST_SIG "\"; " - "extern int __afl_connected;" + "extern __attribute__((visibility(\"default\"))) int __afl_connected;" #ifdef __APPLE__ "__attribute__((visibility(\"default\"))) " "int _L(unsigned int) __asm__(\"___afl_persistent_loop\"); " @@ -2311,6 +2311,11 @@ void add_runtime(aflcc_state_t *aflcc) { } + #if __AFL_CODE_COVERAGE + // Required for dladdr used in afl-compiler-rt.o + insert_param(aflcc, "-ldl"); + #endif + #if !defined(__APPLE__) && !defined(__sun) if (!aflcc->shared_linking && !aflcc->partial_linking) insert_object(aflcc, "dynamic_list.txt", "-Wl,--dynamic-list=%s", 0); diff --git a/utils/dynamic_covfilter/README.md b/utils/dynamic_covfilter/README.md index a64836f1..381e0855 100644 --- a/utils/dynamic_covfilter/README.md +++ b/utils/dynamic_covfilter/README.md @@ -7,7 +7,12 @@ where doing this dynamically without requiring a new build can be beneficial. Especially when dealing with larger builds, it is much more convenient to select the target code locations at runtime instead of doing so at build time. -There are two ways of doing this in AFL++: +There are two ways of doing this in AFL++. Both approaches require a build of +AFL++ with `CODE_COVERAGE=1`, so make sure to build AFL++ first by invoking + +`CODE_COVERAGE=1 make` + +Once you have built AFL++, you can choose out of two approaches: ## Simple Selection with `AFL_PC_FILTER` -- cgit 1.4.1 From eda770fd32b804e3ebd6a43738c0002f6118a463 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 1 Feb 2024 15:13:07 +0100 Subject: push to stable (#1967) * Output afl-clang-fast stuffs only if necessary (#1912) * afl-cc header * afl-cc common declarations - Add afl-cc-state.c - Strip includes, find_object, debug/be_quiet/have_*/callname setting from afl-cc.c - Use debugf_args in main - Modify execvp stuffs to fit new aflcc struct * afl-cc show usage * afl-cc mode selecting 1. compiler_mode by callname in argv[0] 2. compiler_mode by env "AFL_CC_COMPILER" 3. compiler_mode/instrument_mode by command line options "--afl-..." 4. instrument_mode/compiler_mode by various env vars including "AFL_LLVM_INSTRUMENT" 5. final checking steps 6. print "... - mode: %s-%s\n" 7. determine real argv[0] according to compiler_mode * afl-cc macro defs * afl-cc linking behaviors * afl-cc fsanitize behaviors * afl-cc misc * afl-cc body update * afl-cc all-in-one formated with custom-format.py * nits --------- Co-authored-by: vanhauser-thc * changelog * update grammar mutator * lto llvm 12+ * docs(custom_mutators): fix missing ':' (#1953) * Fix broken LTO mode and response file support (#1948) * Strip `-Wl,-no-undefined` during compilation (#1952) Make the compiler wrapper stripping `-Wl,-no-undefined` in addition to `-Wl,--no-undefined`. Both versions of the flag are accepted by clang and, therefore, used by building systems in the wild (e.g., samba will not build without this fix). * Remove dead code in write_to_testcase (#1955) The custom_mutators_count check in if case is duplicate with if condition. The else case is custom_mutators_count == 0, neither custom_mutator_list iteration nor sent check needed. Signed-off-by: Xeonacid * update qemuafl * WIP: Add ability to generate drcov trace using QEMU backend (#1956) * Document new drcov QEMU plugin * Add link to lightkeeper for QEMU drcov file loading --------- Co-authored-by: Jean-Romain Garnier * code format * changelog * sleep on uid != 0 afl-system-config * fix segv about skip_next, warn on unsupported cases of linking options (#1958) * todos * ensure afl-cc only allows available compiler modes * update grammar mutator * disable aslr on apple * fix for arm64 * help selective instrumentation * typos * macos * add compiler test script * apple fixes * bump nyx submodules (#1963) * fix docs * update changelog * update grammar mutator * improve compiler test script * gcc asan workaround (#1966) * fix github merge fuckup * fix * Fix afl-cc (#1968) - Check if too many cmdline params here, each time before insert a new param. - Check if it is "-fsanitize=..." before we do sth. - Remove improper param_st transfer. * Avoid adding llvmnative instrumentation when linking rust sanitizer runtime (#1969) * Dynamic instrumentation filtering for LLVM native (#1971) * Add two dynamic instrumentation filter methods to runtime * Always use pc-table with native pcguard * Add make_symbol_list.py and README * changelog * todos * new forkserver check * fix * nyx test for CI * improve nyx docs * Fixes to afl-cc and documentation (#1974) * Always compile with -ldl when building for CODE_COVERAGE When building with CODE_COVERAGE, the afl runtime contains code that calls `dladdr` which requires -ldl. Under most circumstances, clang already adds this (e.g. when building with pc-table), but there are some circumstances where it isn't added automatically. * Add visibility declaration to __afl_connected When building with hidden visibility, the use of __AFL_LOOP inside such code can cause linker errors due to __afl_connected being declared "hidden". * Update docs to clarify that CODE_COVERAGE=1 is required for dynamic_covfilter * nits * nyx build script updates * test error output * debug ci * debug ci * Improve afl-cc (#1975) * update response file support - full support of rsp file - fix some segv issues * Improve afl-cc - remove dead code about allow/denylist options of sancov - missing `if (!aflcc->have_msan)` - add docs for each function - typo * enable nyx * debug ci * debug ci * debug ci * debug ci * debug ci * debug ci * debug ci * debug ci * fix ci * clean test script * NO_NYX * NO_NYX * fix ci * debug ci * fix ci * finalize ci fix --------- Signed-off-by: Xeonacid Co-authored-by: Sonic <50692172+SonicStark@users.noreply.github.com> Co-authored-by: Xeonacid Co-authored-by: Nils Bars Co-authored-by: Jean-Romain Garnier <7504819+JRomainG@users.noreply.github.com> Co-authored-by: Jean-Romain Garnier Co-authored-by: Sergej Schumilo Co-authored-by: Christian Holler (:decoder) --- .github/workflows/ci.yml | 10 +- Dockerfile | 6 +- GNUmakefile | 20 +- TODO.md | 19 +- afl-cmin | 6 +- custom_mutators/grammar_mutator/GRAMMAR_VERSION | 2 +- custom_mutators/grammar_mutator/grammar_mutator | 2 +- docs/Changelog.md | 7 +- docs/resources/1_instrument_target.drawio.svg | 2 +- instrumentation/README.lto.md | 12 +- instrumentation/SanitizerCoveragePCGUARD.so.cc | 7 + instrumentation/afl-compiler-rt.o.c | 279 ++++++++- instrumentation/afl-llvm-common.cc | 4 +- nyx_mode/LIBNYX_VERSION | 2 +- nyx_mode/QEMU-Nyx | 2 +- nyx_mode/QEMU_NYX_VERSION | 2 +- nyx_mode/README.md | 22 +- nyx_mode/build_nyx_support.sh | 77 ++- nyx_mode/libnyx | 2 +- nyx_mode/update_ref.sh | 6 +- src/afl-cc.c | 753 +++++++++++++++++------- src/afl-forkserver.c | 6 + src/afl-fuzz-init.c | 6 + src/afl-fuzz.c | 11 +- test/test-all.sh | 2 + test/test-basic.sh | 69 ++- test/test-compilers.sh | 7 + test/test-llvm.sh | 28 +- test/test-nyx-mode.sh | 79 +++ test/test-pre.sh | 2 +- utils/dynamic_covfilter/README.md | 60 ++ utils/dynamic_covfilter/make_symbol_list.py | 73 +++ 32 files changed, 1229 insertions(+), 356 deletions(-) create mode 100755 test/test-compilers.sh create mode 100755 test/test-nyx-mode.sh create mode 100644 utils/dynamic_covfilter/README.md create mode 100644 utils/dynamic_covfilter/make_symbol_list.py (limited to 'utils') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdf618b9..ed382fbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,20 +20,18 @@ jobs: AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 steps: - uses: actions/checkout@v3 + - name: update + run: sudo apt-get update && sudo apt-get upgrade -y - name: debug run: apt-cache search plugin-dev | grep gcc-; echo; apt-cache search clang-format- | grep clang-format- - - name: update - run: sudo apt-get update - # && sudo apt-get upgrade -y - name: install packages - #run: sudo apt-get install -y -m -f --install-suggests build-essential git libtool libtool-bin automake bison libglib2.0-0 clang llvm-dev libc++-dev findutils libcmocka-dev python3-dev python3-setuptools ninja-build python3-pip - run: sudo apt-get install -y -m -f build-essential git libtool libtool-bin automake flex bison libglib2.0-0 clang llvm-dev libc++-dev findutils libcmocka-dev python3-dev python3-setuptools ninja-build python3-pip + run: sudo apt-get install -y -m -f build-essential gcc-10 g++-10 git libtool libtool-bin automake flex bison libglib2.0-0 clang-12 llvm-12-dev libc++-dev findutils libcmocka-dev python3-dev python3-setuptools ninja-build python3-pip gcc-10-plugin-dev - name: compiler installed run: gcc -v; echo; clang -v - name: install gcc plugin run: sudo apt-get install -y -m -f --install-suggests $(readlink /usr/bin/gcc)-plugin-dev - name: build afl++ - run: make distrib ASAN_BUILD=1 NO_NYX=1 + run: export NO_NYX=1; export ASAN_BUILD=1; export LLVM_CONFIG=llvm-config-12; make ASAN_BUILD=1 NO_NYX=1 LLVM_CONFIG=llvm-config-12 distrib - name: run tests run: sudo -E ./afl-system-config; make tests # macos: diff --git a/Dockerfile b/Dockerfile index e1616198..99998a61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ ENV NO_CORESIGHT=1 ENV NO_NYX=1 ### Only change these if you know what you are doing: -# LLVM 15 does not look good so we stay at 14 to still have LTO -ENV LLVM_VERSION=14 +# Current recommended LLVM version is 16 +ENV LLVM_VERSION=16 # GCC 12 is producing compile errors for some targets so we stay at GCC 11 ENV GCC_VERSION=11 @@ -88,7 +88,7 @@ ARG TEST_BUILD RUN sed -i.bak 's/^ -/ /g' GNUmakefile && \ make clean && make distrib && \ - ([ "${TEST_BUILD}" ] || (make install && make clean)) && \ + ([ "${TEST_BUILD}" ] || (make install)) && \ mv GNUmakefile.bak GNUmakefile RUN echo "set encoding=utf-8" > /root/.vimrc && \ diff --git a/GNUmakefile b/GNUmakefile index b67f9c15..283c57c2 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -66,6 +66,10 @@ ifdef MSAN_BUILD override LDFLAGS += -fsanitize=memory endif +ifdef CODE_COVERAGE + override CFLAGS += -D__AFL_CODE_COVERAGE=1 +endif + ifeq "$(findstring android, $(shell $(CC) --version 2>/dev/null))" "" ifeq "$(shell echo 'int main() {return 0; }' | $(CC) $(CFLAGS) -Werror -x c - -flto=full -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" CFLAGS_FLTO ?= -flto=full @@ -395,7 +399,7 @@ help: @echo INTROSPECTION - compile afl-fuzz with mutation introspection @echo NO_PYTHON - disable python support @echo NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for normal fuzzing - @echo NO_UTF - do not use UTF-8 for line rendering in status screen (fallback to G1 box drawing, of vanilla AFL) + @echo "NO_UTF - do not use UTF-8 for line rendering in status screen (fallback to G1 box drawing, of vanilla AFL)" @echo NO_NYX - disable building nyx mode dependencies @echo "NO_CORESIGHT - disable building coresight (arm64 only)" @echo NO_UNICORN_ARM64 - disable building unicorn on arm64 @@ -649,16 +653,16 @@ endif # -$(MAKE) -C utils/plot_ui -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" - ifeq "$(ARCH)" "aarch64" - ifndef NO_CORESIGHT +ifeq "$(ARCH)" "aarch64" + ifndef NO_CORESIGHT -$(MAKE) -C coresight_mode - endif endif - ifeq "$(SYS)" "Linux" - ifndef NO_NYX +endif +ifeq "$(SYS)" "Linux" +ifndef NO_NYX -cd nyx_mode && ./build_nyx_support.sh - endif - endif +endif +endif -cd qemu_mode && sh ./build_qemu_support.sh ifeq "$(ARCH)" "aarch64" ifndef NO_UNICORN_ARM64 diff --git a/TODO.md b/TODO.md index 7cab71e8..f2e3963f 100644 --- a/TODO.md +++ b/TODO.md @@ -2,26 +2,21 @@ ## Must + - UI revamp + - hardened_usercopy=0 page_alloc.shuffle=0 + - add value_profile but only enable after 15 minutes without finds + - cmplog max len, cmplog max items envs? - adapt MOpt to new mutation engine - - Update afl->pending_not_fuzzed for MOpt - - cmplog rtn sanity check on fixed length? + no length 1 + - Update afl->pending_not_fuzzed for MOpt + - cmplog rtn sanity check on fixed length? currently we ignore the length - afl-showmap -f support - afl-fuzz multicore wrapper script - when trimming then perform crash detection - - either -L0 and/or -p mmopt results in zero new coverage + - problem: either -L0 and/or -p mmopt results in zero new coverage ## Should -<<<<<<< Updated upstream - - add value_profile but only enable after 15 minutes without finds? -======= - - afl-showmap -f support - - afl-fuzz multicore wrapper script - - UI revamp - - hardened_usercopy=0 page_alloc.shuffle=0 - - add value_profile but only enable after 15 minutes without finds ->>>>>>> Stashed changes - afl-crash-analysis - support persistent and deferred fork server in afl-showmap? - better autodetection of shifting runtime timeout values diff --git a/afl-cmin b/afl-cmin index 566f157d..4aaf3953 100755 --- a/afl-cmin +++ b/afl-cmin @@ -1,11 +1,15 @@ #!/usr/bin/env sh +SYS=$(uname -s) +test "$SYS" = "Darwin" && { + echo Error: afl-cmin does not work on Apple currently. please use afl-cmin.bash instead. + exit 1 +} export AFL_QUIET=1 export ASAN_OPTIONS=detect_leaks=0 THISPATH=`dirname ${0}` export PATH="${THISPATH}:$PATH" awk -f - -- ${@+"$@"} <<'EOF' #!/usr/bin/awk -f - # awk script to minimize a test corpus of input files # # based on afl-cmin bash script written by Michal Zalewski diff --git a/custom_mutators/grammar_mutator/GRAMMAR_VERSION b/custom_mutators/grammar_mutator/GRAMMAR_VERSION index 2568c6a5..3a019448 100644 --- a/custom_mutators/grammar_mutator/GRAMMAR_VERSION +++ b/custom_mutators/grammar_mutator/GRAMMAR_VERSION @@ -1 +1 @@ -ff4e5a2 +5ed4f8d diff --git a/custom_mutators/grammar_mutator/grammar_mutator b/custom_mutators/grammar_mutator/grammar_mutator index ff4e5a26..5ed4f8d6 160000 --- a/custom_mutators/grammar_mutator/grammar_mutator +++ b/custom_mutators/grammar_mutator/grammar_mutator @@ -1 +1 @@ -Subproject commit ff4e5a265daf5d88c4a636fb6a2c22b1d733db09 +Subproject commit 5ed4f8d6e6524df9670af6b411b13031833d67d2 diff --git a/docs/Changelog.md b/docs/Changelog.md index c681c4e1..720a0689 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -12,20 +12,25 @@ - afl-cc: - large rewrite by @SonicStark which fixes a few corner cases, thanks! - LTO mode now requires llvm 12+ + - workaround for ASAN with gcc_plugin mode - instrumentation: - LLVM 18 support, thanks to @devnexen! - - Injection (SQL, LDAP, XSS) feature now available, see + - Injection (SQL, LDAP, XSS) fuzzing feature now available, see `instrumentation/README.injections.md` how to activate/use/expand. - compcov/LAF-intel: - floating point splitting bug fix by @hexcoder - due a bug in LLVM 17 integer splitting is disabled there! - when splitting floats was selected, integers were always split as well, fixed to require AFL_LLVM_LAF_SPLIT_COMPARES or _ALL as it should + - dynamic instrumentation filtering for LLVM NATIVE, thanks @Mozilla! + see utils/dynamic_covfilter/README.md - qemu_mode: - plugins are now activated by default and a new module is included that produces drcov compatible traces for lighthouse/lightkeeper/... thanks to @JRomainG to submitting! + - updated Nyx checkout (fixes a bug) and some QOL - updated the custom grammar mutator + - document afl-cmin does not work on macOS (but afl-cmin.bash does) ### Version ++4.09c (release) diff --git a/docs/resources/1_instrument_target.drawio.svg b/docs/resources/1_instrument_target.drawio.svg index af6ac397..c93fa2b8 100644 --- a/docs/resources/1_instrument_target.drawio.svg +++ b/docs/resources/1_instrument_target.drawio.svg @@ -1,4 +1,4 @@ -
Instrument target
Instrument target
Required task
Required task
Optional task
Optional task
Select compiler

LTO mode
(clang/clang++ 11+)

LLVM mode
(clang/clang++ 3.8+)

GCC_PLUGIN mode
(gcc 5+)

GCC/CLANG mode
(other)
Select compiler...
Select options

Select options depending on
the compiler:

COMPCOV
(only LLVM & LTO)

CmpLog
(only LLVM & LTO)

selective instrumentation
(LTO, LLVM, GCC_PLUGIN)
Select options...
Select sanitizer

Max. one sanitizer type each
in a fuzzing campaign:

ASAN
CFISAN
LSAN
MSAN
TSAN
UBSAN
Select sanitizer...
Compile target source code

Compile target source code depending on the build system:

configure
CMake
Meson Build System
other
Compile target source code...
Modify target

Create a fuzzing harness
by hand for better efficiency.
Modify target...Viewer does not support full SVG 1.1 \ No newline at end of file +
Instrument target
Instrument target
Required task
Required task
Optional task
Optional task
Select compiler

LTO mode
(clang/clang++ 12+)

LLVM mode
(clang/clang++ 3.8+)

GCC_PLUGIN mode
(gcc 5+)

GCC/CLANG mode
(other)
Select compiler...
Select options

Select options depending on
the compiler:

COMPCOV
(only LLVM & LTO)

CmpLog
(only LLVM & LTO)

selective instrumentation
(LTO, LLVM, GCC_PLUGIN)
Select options...
Select sanitizer

Max. one sanitizer type each
in a fuzzing campaign:

ASAN
CFISAN
LSAN
MSAN
TSAN
UBSAN
Select sanitizer...
Compile target source code

Compile target source code depending on the build system:

configure
CMake
Meson Build System
other
Compile target source code...
Modify target

Create a fuzzing harness
by hand for better efficiency.
Modify target...Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/instrumentation/README.lto.md b/instrumentation/README.lto.md index df59cc2a..bd479c26 100644 --- a/instrumentation/README.lto.md +++ b/instrumentation/README.lto.md @@ -2,7 +2,7 @@ ## TL;DR: -This version requires a LLVM 11 or newer. +This version requires a LLVM 12 or newer. 1. Use afl-clang-lto/afl-clang-lto++ because the resulting binaries run slightly faster and give better coverage. @@ -10,7 +10,7 @@ This version requires a LLVM 11 or newer. 2. You can use it together with COMPCOV, COMPLOG and the instrument file listing features. -3. It only works with LLVM 11 or newer. +3. It only works with LLVM 12 or newer. 4. AUTODICTIONARY feature (see below) @@ -60,7 +60,7 @@ AUTODICTIONARY: 11 strings found [+] Instrumented 12071 locations with no collisions (on average 1046 collisions would be in afl-gcc/afl-clang-fast) (non-hardened mode). ``` -## Getting LLVM 11+ +## Getting LLVM 12+ ### Installing llvm @@ -73,7 +73,7 @@ chmod +x llvm.sh sudo ./llvm.sh 15 all ``` -LLVM 11 to 16 should be available in all current Linux repositories. +LLVM 12 to 18 should be available in all current Linux repositories. ## How to build afl-clang-lto @@ -277,7 +277,7 @@ AS=llvm-as ... afl-clang-lto is still work in progress. Known issues: -* Anything that LLVM 11+ cannot compile, afl-clang-lto cannot compile either - +* Anything that LLVM 12+ cannot compile, afl-clang-lto cannot compile either - obviously. * Anything that does not compile with LTO, afl-clang-lto cannot compile either - obviously. @@ -319,7 +319,7 @@ Still more problems came up though as this only works without bugs from LLVM 9 onwards, and with high optimization the link optimization ruins the instrumented control flow graph. -This is all now fixed with LLVM 11+. The llvm's own linker is now able to load +This is all now fixed with LLVM 12+. The llvm's own linker is now able to load passes and this bypasses all problems we had. Happy end :) diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index aae04bb1..f88ce126 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -627,6 +627,13 @@ void ModuleSanitizerCoverageAFL::instrumentFunction( } + if (debug) { + + fprintf(stderr, "SanitizerCoveragePCGUARD: instrumenting %s in %s\n", + F.getName().str().c_str(), F.getParent()->getName().str().c_str()); + + } + InjectCoverage(F, BlocksToInstrument, IsLeafFunc); // InjectTraceForCmp(F, CmpTraceTargets); // InjectTraceForSwitch(F, SwitchTraceTargets); diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 39a762b6..8e55d6a0 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -22,6 +22,10 @@ #define __USE_GNU #endif #include + +__attribute__((weak)) void __sanitizer_symbolize_pc(void *, const char *fmt, + char *out_buf, + size_t out_buf_size); #endif #ifdef __ANDROID__ @@ -124,8 +128,8 @@ struct afl_module_info_t { uintptr_t base_address; // PC Guard start/stop - u32 start; - u32 stop; + u32 *start; + u32 *stop; // PC Table begin/end const uintptr_t *pcs_beg; @@ -147,6 +151,18 @@ afl_module_info_t *__afl_module_info = NULL; u32 __afl_pcmap_size = 0; uintptr_t *__afl_pcmap_ptr = NULL; + +typedef struct { + + uintptr_t start; + u32 len; + +} FilterPCEntry; + +u32 __afl_filter_pcs_size = 0; +FilterPCEntry *__afl_filter_pcs = NULL; +u8 *__afl_filter_pcs_module = NULL; + #endif // __AFL_CODE_COVERAGE /* 1 if we are running in afl, and the forkserver was started, else 0 */ @@ -1587,15 +1603,116 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *guard) { } #ifdef __AFL_CODE_COVERAGE -void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, - const uintptr_t *pcs_end) { +void afl_read_pc_filter_file(const char *filter_file) { - if (__afl_debug) { + FILE *file; + char ch; + + file = fopen(filter_file, "r"); + if (file == NULL) { + + perror("Error opening file"); + return; + + } + + // Check how many PCs we expect to read + while ((ch = fgetc(file)) != EOF) { + + if (ch == '\n') { __afl_filter_pcs_size++; } + + } + + // Rewind to actually read the PCs + fseek(file, 0, SEEK_SET); + + __afl_filter_pcs = malloc(__afl_filter_pcs_size * sizeof(FilterPCEntry)); + if (!__afl_filter_pcs) { + + perror("Error allocating PC array"); + return; + + } + + for (size_t i = 0; i < __afl_filter_pcs_size; i++) { + + fscanf(file, "%lx", &(__afl_filter_pcs[i].start)); + ch = fgetc(file); // Read tab + fscanf(file, "%u", &(__afl_filter_pcs[i].len)); + ch = fgetc(file); // Read tab + + if (!__afl_filter_pcs_module) { + + // Read the module name and store it. + // TODO: We only support one module here right now although + // there is technically no reason to support multiple modules + // in one go. + size_t max_module_len = 255; + size_t i = 0; + __afl_filter_pcs_module = malloc(max_module_len); + while (i < max_module_len - 1 && + (__afl_filter_pcs_module[i] = fgetc(file)) != '\t') { + + ++i; + + } - fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init called\n"); + __afl_filter_pcs_module[i] = '\0'; + fprintf(stderr, "DEBUGXXX: Read module name %s\n", + __afl_filter_pcs_module); + + } + + while ((ch = fgetc(file)) != '\n' && ch != EOF) + ; + + } + + fclose(file); + +} + +u32 locate_in_pcs(uintptr_t needle, u32 *index) { + + size_t lower_bound = 0; + size_t upper_bound = __afl_filter_pcs_size - 1; + + while (lower_bound < __afl_filter_pcs_size && lower_bound <= upper_bound) { + + size_t current_index = lower_bound + (upper_bound - lower_bound) / 2; + + if (__afl_filter_pcs[current_index].start <= needle) { + + if (__afl_filter_pcs[current_index].start + + __afl_filter_pcs[current_index].len > + needle) { + + // Hit + *index = current_index; + return 1; + + } else { + + lower_bound = current_index + 1; + + } + + } else { + + if (!current_index) { break; } + upper_bound = current_index - 1; + + } } + return 0; + +} + +void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + const uintptr_t *pcs_end) { + // If for whatever reason, we cannot get dlinfo here, then pc_guard_init also // couldn't get it and we'd end up attributing to the wrong module. Dl_info dlinfo; @@ -1608,6 +1725,16 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, } + if (__afl_debug) { + + fprintf( + stderr, + "DEBUG: (%u) __sanitizer_cov_pcs_init called for module %s with %ld " + "PCs\n", + getpid(), dlinfo.dli_fname, pcs_end - pcs_beg); + + } + afl_module_info_t *last_module_info = __afl_module_info; while (last_module_info && last_module_info->next) { @@ -1623,34 +1750,78 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, } + if (strcmp(dlinfo.dli_fname, last_module_info->name)) { + + // This can happen with modules being loaded after the forkserver + // where we decide to not track the module. In that case we must + // not track it here either. + fprintf( + stderr, + "WARNING: __sanitizer_cov_pcs_init module info mismatch: %s vs %s\n", + dlinfo.dli_fname, last_module_info->name); + return; + + } + last_module_info->pcs_beg = pcs_beg; last_module_info->pcs_end = pcs_end; + // This is a direct filter based on symbolizing inside the runtime. + // It should only be used with smaller binaries to avoid long startup + // times. Currently, this only supports a single token to scan for. + const char *pc_filter = getenv("AFL_PC_FILTER"); + + // This is a much faster PC filter based on pre-symbolized input data + // that is sorted for fast lookup through binary search. This method + // of filtering is suitable even for very large binaries. + const char *pc_filter_file = getenv("AFL_PC_FILTER_FILE"); + if (pc_filter_file && !__afl_filter_pcs) { + + afl_read_pc_filter_file(pc_filter_file); + + } + // Now update the pcmap. If this is the last module coming in, after all // pre-loaded code, then this will also map all of our delayed previous // modules. - - if (!__afl_pcmap_ptr) { return; } - + // for (afl_module_info_t *mod_info = __afl_module_info; mod_info; mod_info = mod_info->next) { if (mod_info->mapped) { continue; } + if (!mod_info->start) { + + fprintf(stderr, + "ERROR: __sanitizer_cov_pcs_init called with mod_info->start == " + "NULL (%s)\n", + mod_info->name); + abort(); + + } + PCTableEntry *start = (PCTableEntry *)(mod_info->pcs_beg); PCTableEntry *end = (PCTableEntry *)(mod_info->pcs_end); + if (!*mod_info->stop) { continue; } + u32 in_module_index = 0; while (start < end) { - if (mod_info->start + in_module_index >= __afl_map_size) { + if (*mod_info->start + in_module_index >= __afl_map_size) { - fprintf(stderr, "ERROR: __sanitizer_cov_pcs_init out of bounds?!\n"); + fprintf(stderr, + "ERROR: __sanitizer_cov_pcs_init out of bounds?! Start: %u " + "Stop: %u Map Size: %u (%s)\n", + *mod_info->start, *mod_info->stop, __afl_map_size, + mod_info->name); abort(); } + u32 orig_start_index = *mod_info->start; + uintptr_t PC = start->PC; // This is what `GetPreviousInstructionPc` in sanitizer runtime does @@ -1660,7 +1831,58 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, // Calculate relative offset in module PC = PC - mod_info->base_address; - __afl_pcmap_ptr[mod_info->start + in_module_index] = PC; + if (__afl_pcmap_ptr) { + + __afl_pcmap_ptr[orig_start_index + in_module_index] = PC; + + } + + if (pc_filter) { + + char PcDescr[1024]; + // This function is a part of the sanitizer run-time. + // To use it, link with AddressSanitizer or other sanitizer. + __sanitizer_symbolize_pc((void *)start->PC, "%p %F %L", PcDescr, + sizeof(PcDescr)); + + if (strstr(PcDescr, pc_filter)) { + + if (__afl_debug) + fprintf( + stderr, + "DEBUG: Selective instrumentation match: %s (PC %p Index %u)\n", + PcDescr, (void *)start->PC, + *(mod_info->start + in_module_index)); + // No change to guard needed + + } else { + + // Null out the guard to disable this edge + *(mod_info->start + in_module_index) = 0; + + } + + } + + if (__afl_filter_pcs && strstr(mod_info->name, __afl_filter_pcs_module)) { + + u32 result_index; + if (locate_in_pcs(PC, &result_index)) { + + if (__afl_debug) + fprintf(stderr, + "DEBUG: Selective instrumentation match: (PC %lx File " + "Index %u PC Index %u)\n", + PC, result_index, in_module_index); + + } else { + + // Null out the guard to disable this edge + *(mod_info->start + in_module_index) = 0; + + } + + } start++; in_module_index++; @@ -1671,8 +1893,10 @@ void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, if (__afl_debug) { - fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init initialized %u PCs\n", - in_module_index); + fprintf(stderr, + "DEBUG: __sanitizer_cov_pcs_init successfully mapped %s with %u " + "PCs\n", + mod_info->name, in_module_index); } @@ -1706,9 +1930,9 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { fprintf( stderr, "DEBUG: Running __sanitizer_cov_trace_pc_guard_init: %p-%p (%lu edges) " - "after_fs=%u\n", + "after_fs=%u *start=%u\n", start, stop, (unsigned long)(stop - start), - __afl_already_initialized_forkserver); + __afl_already_initialized_forkserver, *start); } @@ -1740,8 +1964,8 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { mod_info->id = last_module_info ? last_module_info->id + 1 : 0; mod_info->name = strdup(dlinfo.dli_fname); mod_info->base_address = (uintptr_t)dlinfo.dli_fbase; - mod_info->start = 0; - mod_info->stop = 0; + mod_info->start = NULL; + mod_info->stop = NULL; mod_info->pcs_beg = NULL; mod_info->pcs_end = NULL; mod_info->mapped = 0; @@ -1757,8 +1981,12 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { } - fprintf(stderr, "[pcmap] Module: %s Base Address: %p\n", dlinfo.dli_fname, - dlinfo.dli_fbase); + if (__afl_debug) { + + fprintf(stderr, "[pcmap] Module: %s Base Address: %p\n", + dlinfo.dli_fname, dlinfo.dli_fbase); + + } } @@ -1861,12 +2089,17 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { #ifdef __AFL_CODE_COVERAGE if (mod_info) { - mod_info->start = *orig_start; - mod_info->stop = *(stop - 1); + if (!mod_info->start) { + + mod_info->start = orig_start; + mod_info->stop = stop - 1; + + } + if (__afl_debug) { fprintf(stderr, "DEBUG: [pcmap] Start Index: %u Stop Index: %u\n", - mod_info->start, mod_info->stop); + *(mod_info->start), *(mod_info->stop)); } diff --git a/instrumentation/afl-llvm-common.cc b/instrumentation/afl-llvm-common.cc index 96952bd6..8e9e7800 100644 --- a/instrumentation/afl-llvm-common.cc +++ b/instrumentation/afl-llvm-common.cc @@ -201,7 +201,7 @@ void initInstrumentList() { if (debug) DEBUGF("loaded allowlist with %zu file and %zu function entries\n", - allowListFiles.size(), allowListFunctions.size()); + allowListFiles.size() / 4, allowListFunctions.size() / 4); } @@ -276,7 +276,7 @@ void initInstrumentList() { if (debug) DEBUGF("loaded denylist with %zu file and %zu function entries\n", - denyListFiles.size(), denyListFunctions.size()); + denyListFiles.size() / 4, denyListFunctions.size() / 4); } diff --git a/nyx_mode/LIBNYX_VERSION b/nyx_mode/LIBNYX_VERSION index da3939ad..9aae19be 100644 --- a/nyx_mode/LIBNYX_VERSION +++ b/nyx_mode/LIBNYX_VERSION @@ -1 +1 @@ -512058a +6833d23 diff --git a/nyx_mode/QEMU-Nyx b/nyx_mode/QEMU-Nyx index 02a6f2ae..1def26f8 160000 --- a/nyx_mode/QEMU-Nyx +++ b/nyx_mode/QEMU-Nyx @@ -1 +1 @@ -Subproject commit 02a6f2aed360cfe76bb3d788dafe517c350d74e5 +Subproject commit 1def26f83e83556d767754581fa52081ffb54b09 diff --git a/nyx_mode/QEMU_NYX_VERSION b/nyx_mode/QEMU_NYX_VERSION index 4f58054c..cac32d41 100644 --- a/nyx_mode/QEMU_NYX_VERSION +++ b/nyx_mode/QEMU_NYX_VERSION @@ -1 +1 @@ -02a6f2aed3 +1def26f83e diff --git a/nyx_mode/README.md b/nyx_mode/README.md index aee9879e..7a2a8e6c 100644 --- a/nyx_mode/README.md +++ b/nyx_mode/README.md @@ -84,9 +84,17 @@ Then the final step: we generate the Nyx package configuration: python3 nyx_mode/packer/packer/nyx_config_gen.py PACKAGE-DIRECTORY Kernel ``` - ## Fuzzing with Nyx mode +Note that you need to load the kvm kernel modules for Nyx: +``` +sudo modprobe -r kvm-intel +sudo modprobe -r kvm +sudo modprobe kvm enable_vmware_backdoor=y +sudo modprobe kvm-intel +cat /sys/module/kvm/parameters/enable_vmware_backdoor | grep -q Y && echi OK || echo KVM module problem +``` + All the hard parts are done, fuzzing with Nyx mode is easy - just supply the `PACKAGE-DIRECTORY` as fuzzing target and specify the `-X` option to afl-fuzz: @@ -94,16 +102,8 @@ All the hard parts are done, fuzzing with Nyx mode is easy - just supply the afl-fuzz -i in -o out -X -- ./PACKAGE-DIRECTORY ``` -Most likely your first run will fail because the Linux modules have to be -specially set up, but afl-fuzz will tell you this on startup and how to rectify -the situation: - -``` -sudo modprobe -r kvm-intel # or kvm-amd for AMD processors -sudo modprobe -r kvm -sudo modprobe kvm enable_vmware_backdoor=y -sudo modprobe kvm-intel # or kvm-amd for AMD processors -``` +If you get a forkserver error upon starting then you did not load the Linux +kvm kernel modules, see above. If you want to fuzz in parallel (and you should!), then this has to be done in a special way: diff --git a/nyx_mode/build_nyx_support.sh b/nyx_mode/build_nyx_support.sh index 581a8292..fda4ec12 100755 --- a/nyx_mode/build_nyx_support.sh +++ b/nyx_mode/build_nyx_support.sh @@ -9,6 +9,21 @@ echo echo "[*] Performing basic sanity checks..." +if [ "$CI" = "true" ]; then + + echo "[-] Error: nyx_mode cannot be tested in the Github CI, skipping ..." + exit 0 + +fi + + +if [ -n "$NO_NYX" ]; then + + echo "[-] Error: the NO_NYX environment variable is set, please unset." + exit 0 + +fi + if [ ! "$(uname -s)" = "Linux" ]; then echo "[-] Error: Nyx mode is only available on Linux." @@ -23,11 +38,17 @@ if [ ! "$(uname -m)" = "x86_64" ]; then fi +cargo help > /dev/null 2>&1 || { + echo "[-] Error: Rust is not installed." + exit 0 +} + echo "[*] Making sure all Nyx is checked out" if git status 1>/dev/null 2>&1; then + set +e git submodule init echo "[*] initializing QEMU-Nyx submodule" git submodule update ./QEMU-Nyx 2>/dev/null # ignore errors @@ -35,6 +56,7 @@ if git status 1>/dev/null 2>&1; then git submodule update ./packer 2>/dev/null # ignore errors echo "[*] initializing libnyx submodule" git submodule update ./libnyx 2>/dev/null # ignore errors + set -e else @@ -48,20 +70,57 @@ test -e packer/.git || { echo "[-] packer not checked out, please install git or test -e libnyx/.git || { echo "[-] libnyx not checked out, please install git or check your internet connection." ; exit 1 ; } test -e QEMU-Nyx/.git || { echo "[-] QEMU-Nyx not checked out, please install git or check your internet connection." ; exit 1 ; } -echo "[*] checking packer init.cpio.gz ..." -if [ ! -f "packer/linux_initramfs/init.cpio.gz" ]; then - (cd packer/linux_initramfs/ && sh pack.sh) + +QEMU_NYX_VERSION="$(cat ./QEMU_NYX_VERSION)" +cd "./QEMU-Nyx" || exit 1 +if [ -n "$NO_CHECKOUT" ]; then + echo "[*] Skipping checkout to $QEMU_NYX_VERSION" +else + echo "[*] Checking out $QEMU_NYX_VERSION" + set +e + sh -c 'git stash' 1>/dev/null 2>/dev/null + git pull 1>/dev/null 2>/dev/null + git checkout "$QEMU_NYX_VERSION" || echo Warning: could not check out to commit $QEMU_NYX_VERSION + set -e fi +cd - > /dev/null -echo "[*] Checking libnyx ..." -if [ ! -f "libnyx/libnyx/target/release/liblibnyx.a" ]; then - (cd libnyx/libnyx && cargo build --release) +PACKER_VERSION="$(cat ./PACKER_VERSION)" +cd "./packer" || exit 1 +if [ -n "$NO_CHECKOUT" ]; then + echo "[*] Skipping checkout to $PACKER_VERSION" +else + echo "[*] Checking out $PACKER_VERSION" + set +e + sh -c 'git stash' 1>/dev/null 2>/dev/null + git pull 1>/dev/null 2>/dev/null + git checkout "$PACKER_VERSION" || echo Warning: could not check out to commit $PACKER_VERSION + set -e fi +cd - > /dev/null -echo "[*] Checking QEMU-Nyx ..." -if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then - (cd QEMU-Nyx && ./compile_qemu_nyx.sh static) +LIBNYX_VERSION="$(cat ./LIBNYX_VERSION)" +cd "./libnyx/" || exit 1 +if [ -n "$NO_CHECKOUT" ]; then + echo "[*] Skipping checkout to $LIBNYX_VERSION" +else + echo "[*] Checking out $LIBNYX_VERSION" + set +e + sh -c 'git stash' 1>/dev/null 2>/dev/null + git pull 1>/dev/null 2>/dev/null + git checkout "$LIBNYX_VERSION" || echo Warning: could not check out to commit $LIBNYX_VERSION + set -e fi +cd - > /dev/null + +echo "[*] checking packer init.cpio.gz ..." +(cd packer/linux_initramfs/ && sh pack.sh) + +echo "[*] Checking libnyx ..." +(cd libnyx/libnyx && cargo build --release) + +echo "[*] Checking QEMU-Nyx ..." +(cd QEMU-Nyx && ./compile_qemu_nyx.sh static ) echo "[*] Checking libnyx.so ..." cp libnyx/libnyx/target/release/liblibnyx.so ../libnyx.so diff --git a/nyx_mode/libnyx b/nyx_mode/libnyx index 512058a6..6833d236 160000 --- a/nyx_mode/libnyx +++ b/nyx_mode/libnyx @@ -1 +1 @@ -Subproject commit 512058a68d58b1a90a4e3971b526a955559735bf +Subproject commit 6833d236dfe785a8a23d8c8d79e74c99fa635004 diff --git a/nyx_mode/update_ref.sh b/nyx_mode/update_ref.sh index 898a803f..146a1255 100755 --- a/nyx_mode/update_ref.sh +++ b/nyx_mode/update_ref.sh @@ -41,7 +41,7 @@ cd .. rm "$UC_VERSION_FILE" echo "$NEW_VERSION" > "$UC_VERSION_FILE" -echo "Done. New XXX version is $NEW_VERSION." +echo "Done. New libnyx version is $NEW_VERSION." UC_VERSION_FILE='./PACKER_VERSION' @@ -68,7 +68,7 @@ cd .. rm "$UC_VERSION_FILE" echo "$NEW_VERSION" > "$UC_VERSION_FILE" -echo "Done. New XXX version is $NEW_VERSION." +echo "Done. New packer version is $NEW_VERSION." UC_VERSION_FILE='./QEMU_NYX_VERSION' @@ -95,5 +95,5 @@ cd .. rm "$UC_VERSION_FILE" echo "$NEW_VERSION" > "$UC_VERSION_FILE" -echo "Done. New XXX version is $NEW_VERSION." +echo "Done. New QEMU-Nyx version is $NEW_VERSION." diff --git a/src/afl-cc.c b/src/afl-cc.c index 192c5423..c300ddfc 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -51,7 +51,7 @@ #define MAX_PARAMS_NUM 2048 #endif -/* Global declarations */ +/** Global declarations -----BEGIN----- **/ typedef enum { @@ -170,8 +170,11 @@ typedef struct aflcc_state { u8 have_instr_env, have_gcc, have_clang, have_llvm, have_gcc_plugin, have_lto, have_optimized_pcguard, have_instr_list; - u8 fortify_set, asan_set, x_set, bit_mode, preprocessor_only, have_unroll, - have_o, have_pic, have_c, shared_linking, partial_linking, non_dash; + u8 fortify_set, x_set, bit_mode, preprocessor_only, have_unroll, have_o, + have_pic, have_c, shared_linking, partial_linking, non_dash, have_fp, + have_flto, have_hidden, have_fortify, have_fcf, have_staticasan, + have_rust_asanrt, have_asan, have_msan, have_ubsan, have_lsan, have_tsan, + have_cfisan; // u8 *march_opt; u8 need_aflpplib; @@ -184,25 +187,27 @@ typedef struct aflcc_state { void aflcc_state_init(aflcc_state_t *, u8 *argv0); -/* Try to find a specific runtime we need, the path to obj would be - allocated and returned. Otherwise it returns NULL on fail. */ u8 *find_object(aflcc_state_t *, u8 *obj); void find_built_deps(aflcc_state_t *); -static inline void limit_params(aflcc_state_t *aflcc, u32 add) { +/* Insert param into the new argv, raise error if MAX_PARAMS_NUM exceeded. */ +static inline void insert_param(aflcc_state_t *aflcc, u8 *param) { - if (aflcc->cc_par_cnt + add >= MAX_PARAMS_NUM) + if (unlikely(aflcc->cc_par_cnt + 1 >= MAX_PARAMS_NUM)) FATAL("Too many command line parameters, please increase MAX_PARAMS_NUM."); -} - -static inline void insert_param(aflcc_state_t *aflcc, u8 *param) { - aflcc->cc_params[aflcc->cc_par_cnt++] = param; } +/* + Insert a param which contains path to the object file. It uses find_object to + get the path based on the name `obj`, and then uses a sprintf like method to + format it with `fmt`. If `fmt` is NULL, the inserted arg is same as the path. + If `msg` provided, it should be an error msg raised if the path can't be + found. `obj` must not be NULL. +*/ static inline void insert_object(aflcc_state_t *aflcc, u8 *obj, u8 *fmt, u8 *msg) { @@ -232,6 +237,7 @@ static inline void insert_object(aflcc_state_t *aflcc, u8 *obj, u8 *fmt, } +/* Insert params into the new argv, make clang load the pass. */ static inline void load_llvm_pass(aflcc_state_t *aflcc, u8 *pass) { #if LLVM_MAJOR >= 11 /* use new pass manager */ @@ -292,8 +298,12 @@ void add_lto_linker(aflcc_state_t *); void add_lto_passes(aflcc_state_t *); void add_runtime(aflcc_state_t *); -/* Working state */ +/** Global declarations -----END----- **/ +/* + Init global state struct. We also extract the callname, + check debug options and if in C++ mode here. +*/ void aflcc_state_init(aflcc_state_t *aflcc, u8 *argv0) { // Default NULL/0 is a good start @@ -353,7 +363,7 @@ void aflcc_state_init(aflcc_state_t *aflcc, u8 *argv0) { } /* - in find_object() we look here: + Try to find a specific runtime we need, in here: 1. firstly we check the $AFL_PATH environment variable location if set 2. next we check argv[0] if it has path information and use it @@ -367,7 +377,6 @@ void aflcc_state_init(aflcc_state_t *aflcc, u8 *argv0) { if all these attempts fail - we return NULL and the caller has to decide what to do. Otherwise the path to obj would be allocated and returned. */ - u8 *find_object(aflcc_state_t *aflcc, u8 *obj) { u8 *argv0 = aflcc->argv0; @@ -500,6 +509,10 @@ u8 *find_object(aflcc_state_t *aflcc, u8 *obj) { } +/* + Deduce some info about compiler toolchains in current system, + from the building results of AFL++ +*/ void find_built_deps(aflcc_state_t *aflcc) { char *ptr = NULL; @@ -572,8 +585,9 @@ void find_built_deps(aflcc_state_t *aflcc) { } -/* compiler_mode & instrument_mode selecting */ +/** compiler_mode & instrument_mode selecting -----BEGIN----- **/ +/* Select compiler_mode by callname, such as "afl-clang-fast", etc. */ void compiler_mode_by_callname(aflcc_state_t *aflcc) { if (strncmp(aflcc->callname, "afl-clang-fast", 14) == 0) { @@ -611,30 +625,26 @@ void compiler_mode_by_callname(aflcc_state_t *aflcc) { aflcc->compiler_mode = GCC_PLUGIN; -#if defined(__x86_64__) - } else if (strncmp(aflcc->callname, "afl-gcc", 7) == 0 || strncmp(aflcc->callname, "afl-g++", 7) == 0) { aflcc->compiler_mode = GCC; -#endif - -#if defined(__x86_64__) - } else if (strcmp(aflcc->callname, "afl-clang") == 0 || strcmp(aflcc->callname, "afl-clang++") == 0) { aflcc->compiler_mode = CLANG; -#endif - } } +/* + Select compiler_mode by env AFL_CC_COMPILER. And passthrough mode can be + regarded as a special compiler_mode, so we check for it here, too. +*/ void compiler_mode_by_environ(aflcc_state_t *aflcc) { if (getenv("AFL_PASSTHROUGH") || getenv("AFL_NOOPT")) { @@ -675,22 +685,14 @@ void compiler_mode_by_environ(aflcc_state_t *aflcc) { aflcc->compiler_mode = GCC_PLUGIN; -#if defined(__x86_64__) - } else if (strcasecmp(ptr, "GCC") == 0) { aflcc->compiler_mode = GCC; -#endif - -#if defined(__x86_64__) - } else if (strcasecmp(ptr, "CLANG") == 0) { aflcc->compiler_mode = CLANG; -#endif - } else FATAL("Unknown AFL_CC_COMPILER mode: %s\n", ptr); @@ -699,7 +701,13 @@ void compiler_mode_by_environ(aflcc_state_t *aflcc) { } -// If it can be inferred, instrument_mode would also be set +/* + Select compiler_mode by command line options --afl-... + If it can be inferred, instrument_mode would also be set. + This can supersedes previous result based on callname + or AFL_CC_COMPILER. And "--afl_noopt"/"--afl-noopt" will + be overwritten by "-g". +*/ void compiler_mode_by_cmdline(aflcc_state_t *aflcc, int argc, char **argv) { char *ptr = NULL; @@ -774,22 +782,14 @@ void compiler_mode_by_cmdline(aflcc_state_t *aflcc, int argc, char **argv) { aflcc->compiler_mode = GCC_PLUGIN; -#if defined(__x86_64__) - } else if (strcasecmp(ptr, "GCC") == 0) { aflcc->compiler_mode = GCC; -#endif - -#if defined(__x86_64__) - } else if (strncasecmp(ptr, "CLANG", 5) == 0) { aflcc->compiler_mode = CLANG; -#endif - } else FATAL("Unknown --afl-... compiler mode: %s\n", argv[i]); @@ -800,6 +800,12 @@ void compiler_mode_by_cmdline(aflcc_state_t *aflcc, int argc, char **argv) { } +/* + Select instrument_mode by those envs in old style: + - USE_TRACE_PC, AFL_USE_TRACE_PC, AFL_LLVM_USE_TRACE_PC, AFL_TRACE_PC + - AFL_LLVM_CALLER, AFL_LLVM_CTX, AFL_LLVM_CTX_K + - AFL_LLVM_NGRAM_SIZE +*/ static void instrument_mode_old_environ(aflcc_state_t *aflcc) { if (getenv("AFL_LLVM_INSTRIM") || getenv("INSTRIM") || @@ -859,7 +865,11 @@ static void instrument_mode_old_environ(aflcc_state_t *aflcc) { } -// compiler_mode would also be set if depended by the instrument_mode +/* + Select instrument_mode by env 'AFL_LLVM_INSTRUMENT'. + Previous compiler_mode will be superseded, if required by some + values of instrument_mode. +*/ static void instrument_mode_new_environ(aflcc_state_t *aflcc) { if (!getenv("AFL_LLVM_INSTRUMENT")) { return; } @@ -960,7 +970,6 @@ static void instrument_mode_new_environ(aflcc_state_t *aflcc) { } -#if defined(__x86_64__) if (strcasecmp(ptr2, "gcc") == 0) { if (!aflcc->instrument_mode || aflcc->instrument_mode == INSTRUMENT_GCC) @@ -975,9 +984,6 @@ static void instrument_mode_new_environ(aflcc_state_t *aflcc) { } -#endif - -#if defined(__x86_64__) if (strcasecmp(ptr2, "clang") == 0) { if (!aflcc->instrument_mode || aflcc->instrument_mode == INSTRUMENT_CLANG) @@ -992,8 +998,6 @@ static void instrument_mode_new_environ(aflcc_state_t *aflcc) { } -#endif - if (strncasecmp(ptr2, "ctx-", strlen("ctx-")) == 0 || strncasecmp(ptr2, "kctx-", strlen("c-ctx-")) == 0 || strncasecmp(ptr2, "k-ctx-", strlen("k-ctx-")) == 0) { @@ -1089,6 +1093,11 @@ static void instrument_mode_new_environ(aflcc_state_t *aflcc) { } +/* + Select instrument_mode by envs, the top wrapper. We check + have_instr_env firstly, then call instrument_mode_old_environ + and instrument_mode_new_environ sequentially. +*/ void instrument_mode_by_environ(aflcc_state_t *aflcc) { if (getenv("AFL_LLVM_INSTRUMENT_FILE") || getenv("AFL_LLVM_WHITELIST") || @@ -1112,6 +1121,10 @@ void instrument_mode_by_environ(aflcc_state_t *aflcc) { } +/* + Workaround to ensure CALLER, CTX, K-CTX and NGRAM + instrumentation were used correctly. +*/ static void instrument_opt_mode_exclude(aflcc_state_t *aflcc) { if ((aflcc->instrument_opt_mode & INSTRUMENT_OPT_CTX) && @@ -1147,6 +1160,11 @@ static void instrument_opt_mode_exclude(aflcc_state_t *aflcc) { } +/* + Last step of compiler_mode & instrument_mode selecting. + We have a few of workarounds here, to check any corner cases, + prepare for a series of fallbacks, and raise warnings or errors. +*/ void mode_final_checkout(aflcc_state_t *aflcc, int argc, char **argv) { if (aflcc->instrument_opt_mode && @@ -1180,11 +1198,11 @@ void mode_final_checkout(aflcc_state_t *aflcc, int argc, char **argv) { switch (aflcc->compiler_mode) { case GCC: - if (!aflcc->have_gcc) FATAL("afl-gcc not available on your platform!"); + if (!aflcc->have_gcc) FATAL("afl-gcc is not available on your platform!"); break; case CLANG: if (!aflcc->have_clang) - FATAL("afl-clang not available on your platform!"); + FATAL("afl-clang is not available on your platform!"); break; case LLVM: if (!aflcc->have_llvm) @@ -1351,6 +1369,10 @@ void mode_final_checkout(aflcc_state_t *aflcc, int argc, char **argv) { } +/* + Print welcome message on screen, giving brief notes about + compiler_mode and instrument_mode. +*/ void mode_notification(aflcc_state_t *aflcc) { char *ptr2 = alloc_printf(" + NGRAM-%u", aflcc->ngram_size); @@ -1389,6 +1411,17 @@ void mode_notification(aflcc_state_t *aflcc) { } +/* + Set argv[0] required by execvp. It can be + - specified by env AFL_CXX + - g++ or clang++ + - CLANGPP_BIN or LLVM_BINDIR/clang++ + when in C++ mode, or + - specified by env AFL_CC + - gcc or clang + - CLANG_BIN or LLVM_BINDIR/clang + otherwise. +*/ void add_real_argv0(aflcc_state_t *aflcc) { static u8 llvm_fullpath[PATH_MAX]; @@ -1455,7 +1488,9 @@ void add_real_argv0(aflcc_state_t *aflcc) { } -/* Macro defs for the preprocessor */ +/** compiler_mode & instrument_mode selecting -----END----- **/ + +/** Macro defs for the preprocessor -----BEGIN----- **/ void add_defs_common(aflcc_state_t *aflcc) { @@ -1464,8 +1499,11 @@ void add_defs_common(aflcc_state_t *aflcc) { } -/* See instrumentation/README.instrument_list.md# - 2-selective-instrumentation-with-_afl_coverage-directives */ +/* + __afl_coverage macro defs. See + instrumentation/README.instrument_list.md# + 2-selective-instrumentation-with-_afl_coverage-directives +*/ void add_defs_selective_instr(aflcc_state_t *aflcc) { if (aflcc->plusplus_mode) { @@ -1499,9 +1537,11 @@ void add_defs_selective_instr(aflcc_state_t *aflcc) { } -/* As documented in instrumentation/README.persistent_mode.md, deferred - forkserver initialization and persistent mode are not available in afl-gcc - and afl-clang. */ +/* + Macro defs for persistent mode. As documented in + instrumentation/README.persistent_mode.md, deferred forkserver initialization + and persistent mode are not available in afl-gcc and afl-clang. +*/ void add_defs_persistent_mode(aflcc_state_t *aflcc) { if (aflcc->compiler_mode == GCC || aflcc->compiler_mode == CLANG) return; @@ -1552,7 +1592,7 @@ void add_defs_persistent_mode(aflcc_state_t *aflcc) { "({ static volatile const char *_B __attribute__((used,unused)); " " _B = (const char*)\"" PERSIST_SIG "\"; " - "extern int __afl_connected;" + "extern __attribute__((visibility(\"default\"))) int __afl_connected;" #ifdef __APPLE__ "__attribute__((visibility(\"default\"))) " "int _L(unsigned int) __asm__(\"___afl_persistent_loop\"); " @@ -1580,9 +1620,15 @@ void add_defs_persistent_mode(aflcc_state_t *aflcc) { } -/* Control _FORTIFY_SOURCE */ +/* + Control macro def of _FORTIFY_SOURCE. It will do nothing + if we detect this routine has been called previously, or + the macro already here in these existing args. +*/ void add_defs_fortify(aflcc_state_t *aflcc, u8 action) { + if (aflcc->have_fortify) { return; } + switch (action) { case 1: @@ -1599,8 +1645,11 @@ void add_defs_fortify(aflcc_state_t *aflcc, u8 action) { } + aflcc->have_fortify = 1; + } +/* Macro defs of __AFL_LEAK_CHECK, __AFL_LSAN_ON and __AFL_LSAN_OFF */ void add_defs_lsan_ctrl(aflcc_state_t *aflcc) { insert_param(aflcc, "-includesanitizer/lsan_interface.h"); @@ -1613,7 +1662,9 @@ void add_defs_lsan_ctrl(aflcc_state_t *aflcc) { } -/* About fsanitize (including PCGUARD features) */ +/** Macro defs for the preprocessor -----END----- **/ + +/** About -fsanitize -----BEGIN----- **/ /* For input "-fsanitize=...", it: @@ -1692,26 +1743,59 @@ static u8 fsanitize_fuzzer_comma(char *string) { } +/* + Parse and process possible -fsanitize related args, return PARAM_MISS + if nothing matched. We have 3 main tasks here for these args: + - Check which one of those sanitizers present here. + - Check if libfuzzer present. We need to block the request of enable + libfuzzer, and link harness with our libAFLDriver.a later. + - Check if SanCov allow/denylist options present. We need to try switching + to LLVMNATIVE instead of using our optimized PCGUARD anyway. If we + can't make it finally for various reasons, just drop these options. +*/ param_st parse_fsanitize(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan) { param_st final_ = PARAM_MISS; - if (!strncmp(cur_argv, "-fsanitize-coverage-", 20) && - strstr(cur_argv, "list=")) { - - if (scan) { - - aflcc->have_instr_list = 1; - final_ = PARAM_SCAN; +// MACRO START +#define HAVE_SANITIZER_SCAN_KEEP(v, k) \ + do { \ + \ + if (strstr(cur_argv, "=" STRINGIFY(k)) || \ + strstr(cur_argv, "," STRINGIFY(k))) { \ + \ + if (scan) { \ + \ + aflcc->have_##v = 1; \ + final_ = PARAM_SCAN; \ + \ + } else { \ + \ + final_ = PARAM_KEEP; \ + \ + } \ + \ + } \ + \ + } while (0) - } else { + // MACRO END - final_ = PARAM_KEEP; // may be set to DROP next + if (!strncmp(cur_argv, "-fsanitize=", strlen("-fsanitize="))) { - } + HAVE_SANITIZER_SCAN_KEEP(asan, address); + HAVE_SANITIZER_SCAN_KEEP(msan, memory); + HAVE_SANITIZER_SCAN_KEEP(ubsan, undefined); + HAVE_SANITIZER_SCAN_KEEP(tsan, thread); + HAVE_SANITIZER_SCAN_KEEP(lsan, leak); + HAVE_SANITIZER_SCAN_KEEP(cfisan, cfi); } +#undef HAVE_SANITIZER_SCAN_KEEP + + // We can't use a "else if" there, because some of the following + // matching rules overlap with those in the if-statement above. if (!strcmp(cur_argv, "-fsanitize=fuzzer")) { if (scan) { @@ -1751,44 +1835,27 @@ param_st parse_fsanitize(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan) { } - } else if ((!strncmp(cur_argv, "-fsanitize=fuzzer-", + } else if (!strncmp(cur_argv, "-fsanitize-coverage-", 20) && - strlen("-fsanitize=fuzzer-")) || - !strncmp(cur_argv, "-fsanitize-coverage", - strlen("-fsanitize-coverage"))) && - (strncmp(cur_argv, "sanitize-coverage-allow", - strlen("sanitize-coverage-allow")) && - strncmp(cur_argv, "sanitize-coverage-deny", - strlen("sanitize-coverage-deny")) && - aflcc->instrument_mode != INSTRUMENT_LLVMNATIVE)) { + strstr(cur_argv, "list=")) { if (scan) { + aflcc->have_instr_list = 1; final_ = PARAM_SCAN; } else { - if (!be_quiet) { WARNF("Found '%s' - stripping!", cur_argv); } - final_ = PARAM_DROP; - - } + if (aflcc->instrument_mode != INSTRUMENT_LLVMNATIVE) { - } - - if (!strcmp(cur_argv, "-fsanitize=address") || - !strcmp(cur_argv, "-fsanitize=memory")) { - - if (scan) { + if (!be_quiet) { WARNF("Found '%s' - stripping!", cur_argv); } + final_ = PARAM_DROP; - // "-fsanitize=undefined,address" may be un-treated, but it's OK. - aflcc->asan_set = 1; - final_ = PARAM_SCAN; + } else { - } else { + final_ = PARAM_KEEP; - // It's impossible that final_ is PARAM_DROP before, - // so no checks are needed here. - final_ = PARAM_KEEP; + } } @@ -1800,76 +1867,125 @@ param_st parse_fsanitize(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan) { } +/* + Add params for sanitizers. Here we need to consider: + - Use static runtime for asan, as much as possible. + - ASAN, MSAN, AFL_HARDEN are mutually exclusive. + - Add options if not found there, on request of AFL_USE_ASAN, AFL_USE_MSAN, + etc. + - Update have_* so that functions called after this can have correct context. + However this also means any functions called before should NOT depend on + these have_*, otherwise they may not work as expected. +*/ void add_sanitizers(aflcc_state_t *aflcc, char **envp) { - if (!aflcc->asan_set) { + if (getenv("AFL_USE_ASAN") || aflcc->have_asan) { + + if (getenv("AFL_USE_MSAN") || aflcc->have_msan) + FATAL("ASAN and MSAN are mutually exclusive"); - if (getenv("AFL_USE_ASAN")) { + if (getenv("AFL_HARDEN")) + FATAL("ASAN and AFL_HARDEN are mutually exclusive"); - if (getenv("AFL_USE_MSAN")) FATAL("ASAN and MSAN are mutually exclusive"); + if (aflcc->compiler_mode == GCC_PLUGIN && !aflcc->have_staticasan) { - if (getenv("AFL_HARDEN")) - FATAL("ASAN and AFL_HARDEN are mutually exclusive"); + insert_param(aflcc, "-static-libasan"); - add_defs_fortify(aflcc, 0); - insert_param(aflcc, "-fsanitize=address"); + } - } else if (getenv("AFL_USE_MSAN")) { + add_defs_fortify(aflcc, 0); + if (!aflcc->have_asan) { insert_param(aflcc, "-fsanitize=address"); } + aflcc->have_asan = 1; - if (getenv("AFL_USE_ASAN")) FATAL("ASAN and MSAN are mutually exclusive"); + } else if (getenv("AFL_USE_MSAN") || aflcc->have_msan) { - if (getenv("AFL_HARDEN")) - FATAL("MSAN and AFL_HARDEN are mutually exclusive"); + if (getenv("AFL_USE_ASAN") || aflcc->have_asan) + FATAL("ASAN and MSAN are mutually exclusive"); - add_defs_fortify(aflcc, 0); - insert_param(aflcc, "-fsanitize=memory"); + if (getenv("AFL_HARDEN")) + FATAL("MSAN and AFL_HARDEN are mutually exclusive"); - } + add_defs_fortify(aflcc, 0); + if (!aflcc->have_msan) { insert_param(aflcc, "-fsanitize=memory"); } + aflcc->have_msan = 1; } - if (getenv("AFL_USE_UBSAN")) { + if (getenv("AFL_USE_UBSAN") || aflcc->have_ubsan) { + + if (!aflcc->have_ubsan) { + + insert_param(aflcc, "-fsanitize=undefined"); + insert_param(aflcc, "-fsanitize-undefined-trap-on-error"); + insert_param(aflcc, "-fno-sanitize-recover=all"); + + } + + if (!aflcc->have_fp) { - insert_param(aflcc, "-fsanitize=undefined"); - insert_param(aflcc, "-fsanitize-undefined-trap-on-error"); - insert_param(aflcc, "-fno-sanitize-recover=all"); - insert_param(aflcc, "-fno-omit-frame-pointer"); + insert_param(aflcc, "-fno-omit-frame-pointer"); + aflcc->have_fp = 1; + + } + + aflcc->have_ubsan = 1; } - if (getenv("AFL_USE_TSAN")) { + if (getenv("AFL_USE_TSAN") || aflcc->have_tsan) { + + if (!aflcc->have_fp) { + + insert_param(aflcc, "-fno-omit-frame-pointer"); + aflcc->have_fp = 1; + + } - insert_param(aflcc, "-fsanitize=thread"); - insert_param(aflcc, "-fno-omit-frame-pointer"); + if (!aflcc->have_tsan) { insert_param(aflcc, "-fsanitize=thread"); } + aflcc->have_tsan = 1; } - if (getenv("AFL_USE_LSAN")) { + if (getenv("AFL_USE_LSAN") && !aflcc->have_lsan) { insert_param(aflcc, "-fsanitize=leak"); add_defs_lsan_ctrl(aflcc); + aflcc->have_lsan = 1; } - if (getenv("AFL_USE_CFISAN")) { + if (getenv("AFL_USE_CFISAN") || aflcc->have_cfisan) { if (aflcc->compiler_mode == GCC_PLUGIN || aflcc->compiler_mode == GCC) { - insert_param(aflcc, "-fcf-protection=full"); + if (!aflcc->have_fcf) { insert_param(aflcc, "-fcf-protection=full"); } } else { - if (!aflcc->lto_mode) { + if (!aflcc->lto_mode && !aflcc->have_flto) { uint32_t i = 0, found = 0; - while (envp[i] != NULL && !found) + while (envp[i] != NULL && !found) { + if (strncmp("-flto", envp[i++], 5) == 0) found = 1; - if (!found) insert_param(aflcc, "-flto"); + + } + + if (!found) { insert_param(aflcc, "-flto"); } + aflcc->have_flto = 1; } - insert_param(aflcc, "-fsanitize=cfi"); - insert_param(aflcc, "-fvisibility=hidden"); + if (!aflcc->have_cfisan) { insert_param(aflcc, "-fsanitize=cfi"); } + + if (!aflcc->have_hidden) { + + insert_param(aflcc, "-fvisibility=hidden"); + aflcc->have_hidden = 1; + + } + + aflcc->have_cfisan = 1; } @@ -1877,42 +1993,48 @@ void add_sanitizers(aflcc_state_t *aflcc, char **envp) { } +/* Add params to enable LLVM SanCov, the native PCGUARD */ void add_native_pcguard(aflcc_state_t *aflcc) { + /* If there is a rust ASan runtime on the command line, it is likely we're + * linking from rust and adding native flags requiring the sanitizer runtime + * will trigger native clang to add yet another runtime, causing linker + * errors. For now we shouldn't add instrumentation here, we're linking + * anyway. + */ + if (aflcc->have_rust_asanrt) { return; } + /* If llvm-config doesn't figure out LLVM_MAJOR, just go on anyway and let compiler complain if doesn't work. */ - if (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { - #if LLVM_MAJOR > 0 && LLVM_MAJOR < 6 - FATAL("pcguard instrumentation with pc-table requires LLVM 6.0.1+"); + FATAL("pcguard instrumentation with pc-table requires LLVM 6.0.1+"); #else #if LLVM_MAJOR == 0 - WARNF( - "pcguard instrumentation with pc-table requires LLVM 6.0.1+" - " otherwise the compiler will fail"); + WARNF( + "pcguard instrumentation with pc-table requires LLVM 6.0.1+" + " otherwise the compiler will fail"); #endif + if (aflcc->instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { + insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard,bb,no-prune,pc-table"); -#endif } else { -#if LLVM_MAJOR > 0 && LLVM_MAJOR < 4 - FATAL("pcguard instrumentation requires LLVM 4.0.1+"); -#else - #if LLVM_MAJOR == 0 - WARNF( - "pcguard instrumentation requires LLVM 4.0.1+" - " otherwise the compiler will fail"); - #endif - insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard"); -#endif + insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard,pc-table"); } +#endif + } +/* + Add params to launch our optimized PCGUARD on request. + It will fallback to use the native PCGUARD in some cases. If so, plz + bear in mind that instrument_mode will be set to INSTRUMENT_LLVMNATIVE. +*/ void add_optimized_pcguard(aflcc_state_t *aflcc) { #if LLVM_MAJOR >= 13 @@ -1929,7 +2051,7 @@ void add_optimized_pcguard(aflcc_state_t *aflcc) { SAYF( "Using unoptimized trace-pc-guard, due usage of " "-fsanitize-coverage-allow/denylist, you can use " - "AFL_LLVM_ALLOWLIST/AFL_LLMV_DENYLIST instead.\n"); + "AFL_LLVM_ALLOWLIST/AFL_LLVM_DENYLIST instead.\n"); insert_param(aflcc, "-fsanitize-coverage=trace-pc-guard"); aflcc->instrument_mode = INSTRUMENT_LLVMNATIVE; @@ -1964,8 +2086,14 @@ void add_optimized_pcguard(aflcc_state_t *aflcc) { } -/* Linking behaviors */ +/** About -fsanitize -----END----- **/ +/** Linking behaviors -----BEGIN----- **/ + +/* + Parse and process possible linking stage related args, + return PARAM_MISS if nothing matched. +*/ param_st parse_linking_params(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan, u8 *skip_next, char **argv) { @@ -2128,6 +2256,7 @@ param_st parse_linking_params(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan, } +/* Add params to specify the linker used in LTO */ void add_lto_linker(aflcc_state_t *aflcc) { unsetenv("AFL_LD"); @@ -2167,6 +2296,7 @@ void add_lto_linker(aflcc_state_t *aflcc) { } +/* Add params to launch SanitizerCoverageLTO.so when linking */ void add_lto_passes(aflcc_state_t *aflcc) { #if defined(AFL_CLANG_LDPATH) && LLVM_MAJOR >= 15 @@ -2185,6 +2315,7 @@ void add_lto_passes(aflcc_state_t *aflcc) { } +/* Add params to link with libAFLDriver.a on request */ static void add_aflpplib(aflcc_state_t *aflcc) { if (!aflcc->need_aflpplib) return; @@ -2220,6 +2351,7 @@ static void add_aflpplib(aflcc_state_t *aflcc) { } +/* Add params to link with runtimes depended by our instrumentation */ void add_runtime(aflcc_state_t *aflcc) { if (aflcc->preprocessor_only || aflcc->have_c || !aflcc->non_dash) { @@ -2281,6 +2413,11 @@ void add_runtime(aflcc_state_t *aflcc) { } + #if __AFL_CODE_COVERAGE + // Required for dladdr used in afl-compiler-rt.o + insert_param(aflcc, "-ldl"); + #endif + #if !defined(__APPLE__) && !defined(__sun) if (!aflcc->shared_linking && !aflcc->partial_linking) insert_object(aflcc, "dynamic_list.txt", "-Wl,--dynamic-list=%s", 0); @@ -2310,8 +2447,14 @@ void add_runtime(aflcc_state_t *aflcc) { } -/* Misc */ +/** Linking behaviors -----END----- **/ +/** Miscellaneous routines -----BEGIN----- **/ + +/* + Add params to make compiler driver use our afl-as + as assembler, required by the vanilla instrumentation. +*/ void add_assembler(aflcc_state_t *aflcc) { u8 *afl_as = find_object(aflcc, "as"); @@ -2328,6 +2471,7 @@ void add_assembler(aflcc_state_t *aflcc) { } +/* Add params to launch the gcc plugins for instrumentation. */ void add_gcc_plugin(aflcc_state_t *aflcc) { if (aflcc->cmplog_mode) { @@ -2344,6 +2488,7 @@ void add_gcc_plugin(aflcc_state_t *aflcc) { } +/* Add some miscellaneous params required by our instrumentation. */ void add_misc_params(aflcc_state_t *aflcc) { if (getenv("AFL_NO_BUILTIN") || getenv("AFL_LLVM_LAF_TRANSFORM_COMPARES") || @@ -2390,6 +2535,10 @@ void add_misc_params(aflcc_state_t *aflcc) { } +/* + Parse and process a variety of args under our matching rules, + return PARAM_MISS if nothing matched. +*/ param_st parse_misc_params(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan) { param_st final_ = PARAM_MISS; @@ -2447,6 +2596,36 @@ param_st parse_misc_params(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan) { SCAN_KEEP(aflcc->have_c, 1); + } else if (!strcmp(cur_argv, "-static-libasan")) { + + SCAN_KEEP(aflcc->have_staticasan, 1); + + } else if (strstr(cur_argv, "librustc") && strstr(cur_argv, "_rt.asan.a")) { + + SCAN_KEEP(aflcc->have_rust_asanrt, 1); + + } else if (!strcmp(cur_argv, "-fno-omit-frame-pointer")) { + + SCAN_KEEP(aflcc->have_fp, 1); + + } else if (!strcmp(cur_argv, "-fvisibility=hidden")) { + + SCAN_KEEP(aflcc->have_hidden, 1); + + } else if (!strcmp(cur_argv, "-flto") || !strcmp(cur_argv, "-flto=full")) { + + SCAN_KEEP(aflcc->have_flto, 1); + + } else if (!strncmp(cur_argv, "-D_FORTIFY_SOURCE", + + strlen("-D_FORTIFY_SOURCE"))) { + + SCAN_KEEP(aflcc->have_fortify, 1); + + } else if (!strncmp(cur_argv, "-fcf-protection", strlen("-fcf-protection"))) { + + SCAN_KEEP(aflcc->have_cfisan, 1); + } else if (!strncmp(cur_argv, "-O", 2)) { SCAN_KEEP(aflcc->have_o, 1); @@ -2510,6 +2689,9 @@ param_st parse_misc_params(aflcc_state_t *aflcc, u8 *cur_argv, u8 scan) { } +/** Miscellaneous routines -----END----- **/ + +/* Print help message on request */ static void maybe_usage(aflcc_state_t *aflcc, int argc, char **argv) { if (argc < 2 || strncmp(argv[1], "-h", 2) == 0) { @@ -2538,11 +2720,11 @@ static void maybe_usage(aflcc_state_t *aflcc, int argc, char **argv) { "MODES: NCC PERSIST DICT LAF " "CMPLOG SELECT\n" " [LLVM] LLVM: %s%s\n" - " PCGUARD %s yes yes module yes yes " + " PCGUARD %s yes yes module yes yes " "yes\n" - " NATIVE AVAILABLE no yes no no " + " NATIVE AVAILABLE no yes no no " "part. yes\n" - " CLASSIC %s no yes module yes yes " + " CLASSIC %s no yes module yes yes " "yes\n" " - NORMAL\n" " - CALLER\n" @@ -2805,11 +2987,27 @@ static void maybe_usage(aflcc_state_t *aflcc, int argc, char **argv) { } +/* + Process params passed to afl-cc. + + We have two working modes, *scan* and *non-scan*. In scan mode, + the main task is to set some variables in aflcc according to current argv[i], + while in non-scan mode, is to choose keep or drop current argv[i]. + + We have several matching routines being called sequentially in the while-loop, + and each of them try to parse and match current argv[i] according to their own + rules. If one miss match, the next will then take over. In non-scan mode, each + argv[i] mis-matched by all the routines will be kept. + + These routines are: + 1. parse_misc_params + 2. parse_fsanitize + 3. parse_linking_params + 4. `if (*cur == '@') {...}`, i.e., parse response files +*/ static void process_params(aflcc_state_t *aflcc, u8 scan, u32 argc, char **argv) { - limit_params(aflcc, argc); - // for (u32 x = 0; x < argc; ++x) fprintf(stderr, "[%u] %s\n", x, argv[x]); /* Process the argument list. */ @@ -2833,134 +3031,249 @@ static void process_params(aflcc_state_t *aflcc, u8 scan, u32 argc, if (PARAM_MISS != parse_linking_params(aflcc, cur, scan, &skip_next, argv)) continue; + /* Response file support -----BEGIN----- + We have two choices - move everything to the command line or + rewrite the response files to temporary files and delete them + afterwards. We choose the first for easiness. + For clang, llvm::cl::ExpandResponseFiles does this, however it + only has C++ interface. And for gcc there is expandargv in libiberty, + written in C, but we can't simply copy-paste since its LGPL licensed. + So here we use an equivalent FSM as alternative, and try to be compatible + with the two above. See: + - https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html + - driver::expand_at_files in gcc.git/gcc/gcc.c + - expandargv in gcc.git/libiberty/argv.c + - llvm-project.git/clang/tools/driver/driver.cpp + - ExpandResponseFiles in + llvm-project.git/llvm/lib/Support/CommandLine.cpp + */ if (*cur == '@') { - // response file support. - // we have two choices - move everything to the command line or - // rewrite the response files to temporary files and delete them - // afterwards. We choose the first for easiness. - // We do *not* support quotes in the rsp files to cope with spaces in - // filenames etc! If you need that then send a patch! u8 *filename = cur + 1; if (aflcc->debug) { DEBUGF("response file=%s\n", filename); } - FILE *f = fopen(filename, "r"); - struct stat st; // Check not found or empty? let the compiler complain if so. - if (!f || fstat(fileno(f), &st) < 0 || st.st_size < 1) { + FILE *f = fopen(filename, "r"); + if (!f) { if (!scan) insert_param(aflcc, cur); continue; } - u8 *tmpbuf = malloc(st.st_size + 2), *ptr; - char **args = malloc(sizeof(char *) * (st.st_size >> 1)); - int count = 1, cont = 0, cont_act = 0; + struct stat st; + if (fstat(fileno(f), &st) || !S_ISREG(st.st_mode) || st.st_size < 1) { - while (fgets(tmpbuf, st.st_size + 1, f)) { + fclose(f); + if (!scan) insert_param(aflcc, cur); + continue; - ptr = tmpbuf; - // fprintf(stderr, "1: %s\n", ptr); - // no leading whitespace - while (isspace(*ptr)) { + } - ++ptr; - cont_act = 0; + // Limit the number of response files, the max value + // just keep consistent with expandargv. Only do this in + // scan mode, and not touch rsp_count anymore in the next. + static u32 rsp_count = 2000; + if (scan) { - } + if (rsp_count == 0) FATAL("Too many response files provided!"); - // no comments, no empty lines - if (*ptr == '#' || *ptr == '\n' || !*ptr) { continue; } - // remove LF - if (ptr[strlen(ptr) - 1] == '\n') { ptr[strlen(ptr) - 1] = 0; } - // remove CR - if (*ptr && ptr[strlen(ptr) - 1] == '\r') { ptr[strlen(ptr) - 1] = 0; } - // handle \ at end of line - if (*ptr && ptr[strlen(ptr) - 1] == '\\') { + --rsp_count; - cont = 1; - ptr[strlen(ptr) - 1] = 0; + } - } + // argc, argv acquired from this rsp file. Note that + // process_params ignores argv[0], we need to put a const "" here. + u32 argc_read = 1; + char **argv_read = ck_alloc(sizeof(char *)); + argv_read[0] = ""; + + char *arg_buf = NULL; + u64 arg_len = 0; + + enum fsm_state { + + fsm_whitespace, // whitespace seen so far + fsm_double_quote, // have unpaired double quote + fsm_single_quote, // have unpaired single quote + fsm_backslash, // a backslash is seen with no unpaired quote + fsm_normal // a normal char is seen + + }; + + // Workaround to append c to arg buffer, and append the buffer to argv +#define ARG_ALLOC(c) \ + do { \ + \ + ++arg_len; \ + arg_buf = ck_realloc(arg_buf, (arg_len + 1) * sizeof(char)); \ + arg_buf[arg_len] = '\0'; \ + arg_buf[arg_len - 1] = (char)c; \ + \ + } while (0) + +#define ARG_STORE() \ + do { \ + \ + ++argc_read; \ + argv_read = ck_realloc(argv_read, argc_read * sizeof(char *)); \ + argv_read[argc_read - 1] = arg_buf; \ + arg_buf = NULL; \ + arg_len = 0; \ + \ + } while (0) - // fprintf(stderr, "2: %s\n", ptr); + int cur_chr = (int)' '; // init as whitespace, as a good start :) + enum fsm_state state_ = fsm_whitespace; - // remove whitespace at end - while (*ptr && isspace(ptr[strlen(ptr) - 1])) { + while (cur_chr != EOF) { - ptr[strlen(ptr) - 1] = 0; - cont = 0; + switch (state_) { - } + case fsm_whitespace: - // fprintf(stderr, "3: %s\n", ptr); - if (*ptr) { + if (arg_buf) { + + ARG_STORE(); + break; + + } - do { + if (isspace(cur_chr)) { - u8 *value = ptr; - while (*ptr && !isspace(*ptr)) { + cur_chr = fgetc(f); - ++ptr; + } else if (cur_chr == (int)'\'') { + + state_ = fsm_single_quote; + cur_chr = fgetc(f); + + } else if (cur_chr == (int)'"') { + + state_ = fsm_double_quote; + cur_chr = fgetc(f); + + } else if (cur_chr == (int)'\\') { + + state_ = fsm_backslash; + cur_chr = fgetc(f); + + } else { + + state_ = fsm_normal; } - while (*ptr && isspace(*ptr)) { + break; + + case fsm_normal: + + if (isspace(cur_chr)) { + + state_ = fsm_whitespace; + + } else if (cur_chr == (int)'\'') { + + state_ = fsm_single_quote; + cur_chr = fgetc(f); + + } else if (cur_chr == (int)'\"') { - *ptr++ = 0; + state_ = fsm_double_quote; + cur_chr = fgetc(f); + + } else if (cur_chr == (int)'\\') { + + state_ = fsm_backslash; + cur_chr = fgetc(f); + + } else { + + ARG_ALLOC(cur_chr); + cur_chr = fgetc(f); } - if (cont_act) { + break; + + case fsm_backslash: + + ARG_ALLOC(cur_chr); + cur_chr = fgetc(f); + state_ = fsm_normal; - u32 len = strlen(args[count - 1]) + strlen(value) + 1; - u8 *tmp = malloc(len); - snprintf(tmp, len, "%s%s", args[count - 1], value); - free(args[count - 1]); - args[count - 1] = tmp; - cont_act = 0; + break; + + case fsm_single_quote: + + if (cur_chr == (int)'\\') { + + cur_chr = fgetc(f); + if (cur_chr == EOF) break; + ARG_ALLOC(cur_chr); + + } else if (cur_chr == (int)'\'') { + + state_ = fsm_normal; } else { - args[count++] = strdup(value); + ARG_ALLOC(cur_chr); } - } while (*ptr); + cur_chr = fgetc(f); + break; - } + case fsm_double_quote: + + if (cur_chr == (int)'\\') { - if (cont) { + cur_chr = fgetc(f); + if (cur_chr == EOF) break; + ARG_ALLOC(cur_chr); - cont_act = 1; - cont = 0; + } else if (cur_chr == (int)'"') { + + state_ = fsm_normal; + + } else { + + ARG_ALLOC(cur_chr); + + } + + cur_chr = fgetc(f); + break; + + default: + break; } } - if (count) { process_params(aflcc, scan, count, args); } + if (arg_buf) { ARG_STORE(); } // save the pending arg after EOF - // we cannot free args[] unless we don't need - // to keep any reference in cc_params - if (scan) { +#undef ARG_ALLOC +#undef ARG_STORE - if (count) do { + if (argc_read > 1) { process_params(aflcc, scan, argc_read, argv_read); } - free(args[--count]); + // We cannot free argv_read[] unless we don't need to keep any + // reference in cc_params. Never free argv[0], the const "". + if (scan) { - } while (count); + while (argc_read > 1) + ck_free(argv_read[--argc_read]); - free(args); + ck_free(argv_read); } - free(tmpbuf); - continue; - } + } /* Response file support -----END----- */ if (!scan) insert_param(aflcc, cur); @@ -2968,8 +3281,7 @@ static void process_params(aflcc_state_t *aflcc, u8 scan, u32 argc, } -/* Copy argv to cc_params, making the necessary edits. */ - +/* Process each of the existing argv, also add a few new args. */ static void edit_params(aflcc_state_t *aflcc, u32 argc, char **argv, char **envp) { @@ -3110,7 +3422,6 @@ static void edit_params(aflcc_state_t *aflcc, u32 argc, char **argv, } /* Main entry point */ - int main(int argc, char **argv, char **envp) { aflcc_state_t *aflcc = malloc(sizeof(aflcc_state_t)); diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 3f9bfa72..214b4fe9 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -1017,6 +1017,12 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (rlen == 4) { + if (status >= 0x41464c00 && status <= 0x41464cff) { + + FATAL("Target uses the new forkserver model, you need to switch to a newer afl-fuzz too!"); + + } + if (!be_quiet) { OKF("All right - fork server is up."); } if (getenv("AFL_DEBUG")) { diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 35932913..8ab44a3b 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -124,6 +124,9 @@ void bind_to_free_cpu(afl_state_t *afl) { } WARNF("Not binding to a CPU core (AFL_NO_AFFINITY set)."); + #ifdef __linux__ + if (afl->fsrv.nyx_mode) { afl->fsrv.nyx_bind_cpu_id = 0; } + #endif return; } @@ -151,6 +154,9 @@ void bind_to_free_cpu(afl_state_t *afl) { } else { OKF("CPU binding request using -b %d successful.", afl->cpu_to_bind); + #ifdef __linux__ + if (afl->fsrv.nyx_mode) { afl->fsrv.nyx_bind_cpu_id = afl->cpu_to_bind; } + #endif } diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 2d5787e8..8cf6c735 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -165,7 +165,7 @@ static void usage(u8 *argv0, int more_help) { "\n" "Mutator settings:\n" - " -a - target input format, \"text\" or \"binary\" (default: " + " -a type - target input format, \"text\" or \"binary\" (default: " "generic)\n" " -g minlength - set min length of generated fuzz input (default: 1)\n" " -G maxlength - set max length of generated fuzz input (default: " @@ -1915,6 +1915,15 @@ int main(int argc, char **argv_orig, char **envp) { bind_to_free_cpu(afl); #endif /* HAVE_AFFINITY */ + #ifdef __linux__ + if (afl->fsrv.nyx_mode && afl->fsrv.nyx_bind_cpu_id == 0xFFFFFFFF) { + + afl->fsrv.nyx_bind_cpu_id = 0; + + } + + #endif + #ifdef __HAIKU__ /* Prioritizes performance over power saving */ set_scheduler_mode(SCHEDULER_MODE_LOW_LATENCY); diff --git a/test/test-all.sh b/test/test-all.sh index 3cb692ca..65cfb812 100755 --- a/test/test-all.sh +++ b/test/test-all.sh @@ -16,6 +16,8 @@ . ./test-frida-mode.sh +. ./test-nyx-mode.sh + . ./test-unicorn-mode.sh . ./test-custom-mutators.sh diff --git a/test/test-basic.sh b/test/test-basic.sh index 61ad4b7c..7005d3ce 100755 --- a/test/test-basic.sh +++ b/test/test-basic.sh @@ -2,6 +2,7 @@ . ./test-pre.sh +OS=$(uname -s) AFL_GCC=afl-gcc $ECHO "$BLUE[*] Testing: ${AFL_GCC}, afl-showmap, afl-fuzz, afl-cmin and afl-tmin" @@ -61,7 +62,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc } # now we want to be sure that afl-fuzz is working # make sure crash reporter is disabled on Mac OS X - (test "$(uname -s)" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { + (test "$OS" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { $ECHO "$RED[!] we cannot run afl-fuzz with enabled crash reporter. Run 'sudo sh afl-system-config'.$RESET" true }) || { @@ -84,16 +85,20 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc } echo 000000000000000000000000 > in/in2 echo 111 > in/in3 - mkdir -p in2 - ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? - CNT=`ls in2/* 2>/dev/null | wc -l` - case "$CNT" in - *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; - *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" - CODE=1 - ;; - esac - rm -f in2/in* + test "$OS" = "Darwin" && { + $ECHO "$GREY[*] afl-cmin not available on macOS, cannot test afl-cmin" + } || { + mkdir -p in2 + ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? + CNT=`ls in2/* 2>/dev/null | wc -l` + case "$CNT" in + *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" + CODE=1 + ;; + esac + rm -f in2/in* + } export AFL_QUIET=1 if command -v bash >/dev/null ; then { ../afl-cmin.bash -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null @@ -182,7 +187,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc } # now we want to be sure that afl-fuzz is working # make sure crash reporter is disabled on Mac OS X - (test "$(uname -s)" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { + (test "$OS" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { $ECHO "$RED[!] we cannot run afl-fuzz with enabled crash reporter. Run 'sudo sh afl-system-config'.$RESET" true }) || { @@ -204,25 +209,29 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc } } echo 000000000000000000000000 > in/in2 - echo AAA > in/in3 - mkdir -p in2 - ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? - CNT=`ls in2/* 2>/dev/null | wc -l` - case "$CNT" in - *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; - \ *1|1) { # allow leading whitecase for portability - test -s in2/* && $ECHO "$YELLOW[?] afl-cmin did minimize to one testcase. This can be a bug or due compiler optimization." - test -s in2/* || { - $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" - CODE=1 + echo AAA > in/in2 + test "$OS" = "Darwin" && { + $ECHO "$GREY[*] afl-cmin not available on macOS, cannot test afl-cmin" + } || { + mkdir -p in2 + ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? + CNT=`ls in2/* 2>/dev/null | wc -l` + case "$CNT" in + *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + \ *1|1) { # allow leading whitecase for portability + test -s in2/* && $ECHO "$YELLOW[?] afl-cmin did minimize to one testcase. This can be a bug or due compiler optimization." + test -s in2/* || { + $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" + CODE=1 + } } - } - ;; - *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" - CODE=1 - ;; - esac - rm -f in2/in* + ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" + CODE=1 + ;; + esac + rm -f in2/in* + } export AFL_QUIET=1 if command -v bash >/dev/null ; then { ../afl-cmin.bash -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null diff --git a/test/test-compilers.sh b/test/test-compilers.sh new file mode 100755 index 00000000..b47cf38d --- /dev/null +++ b/test/test-compilers.sh @@ -0,0 +1,7 @@ +#!/bin/sh +echo Testing compilers ... +for cc in afl-cc afl-gcc afl-clang afl-clang-fast afl-clang-lto afl-gcc-fast; do + test -e ../$cc && { { ../$cc -o t ../test-instr.c >/dev/null 2<&1 && echo Success: $cc ; } || echo Failing: $cc ; } || echo Missing: $cc +done +rm -f t +echo Done! diff --git a/test/test-llvm.sh b/test/test-llvm.sh index 95e43b1c..53bbd7b4 100755 --- a/test/test-llvm.sh +++ b/test/test-llvm.sh @@ -2,6 +2,8 @@ . ./test-pre.sh +OS=$(uname -s) + $ECHO "$BLUE[*] Testing: llvm_mode, afl-showmap, afl-fuzz, afl-cmin and afl-tmin" test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 @@ -123,7 +125,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { } # now we want to be sure that afl-fuzz is working # make sure crash reporter is disabled on Mac OS X - (test "$(uname -s)" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { + (test "$OS" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && { $ECHO "$RED[!] we cannot run afl-fuzz with enabled crash reporter. Run 'sudo sh afl-system-config'.$RESET" CODE=1 true @@ -146,18 +148,22 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { } } test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc" || { + mkdir -p in2 echo 000000000000000000000000 > in/in2 echo 111 > in/in3 - mkdir -p in2 - ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? - CNT=`ls in2/* 2>/dev/null | wc -l` - case "$CNT" in - *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; - *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" - CODE=1 - ;; - esac - rm -f in2/in* + test "$OS" = "Darwin" && { + $ECHO "$GREY[*] afl-cmin not available on macOS, cannot test afl-cmin" + } || { + ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr? + CNT=`ls in2/* 2>/dev/null | wc -l` + case "$CNT" in + *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)" + CODE=1 + ;; + esac + rm -f in2/in* + } export AFL_QUIET=1 if type bash >/dev/null ; then { ../afl-cmin.bash -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null diff --git a/test/test-nyx-mode.sh b/test/test-nyx-mode.sh new file mode 100755 index 00000000..6de63f1b --- /dev/null +++ b/test/test-nyx-mode.sh @@ -0,0 +1,79 @@ +#!/bin/sh + +. ./test-pre.sh + +$ECHO "$BLUE[*] Testing: nyx_mode" + +test "$CI" = "true" && { + $ECHO "$YELLOW[-] nyx_mode cannot be tested in the Github CI, skipping ..." + exit 0 +} + +unset AFL_CC + +test -e ../libnyx.so && { + ../afl-cc -o test-instr ../test-instr.c > errors 2>&1 + test -e test-instr && { + { + rm -rf nyx-test in out + $ECHO "$GREY[*] running nyx_packer" + python3 ../nyx_mode/packer/packer/nyx_packer.py \ + ./test-instr \ + nyx-test \ + afl \ + instrumentation \ + --fast_reload_mode \ + --purge > /dev/null 2>&1 + + test -e nyx-test/test-instr && { + + $ECHO "$GREY[*] running nyx_config_gen" + python3 ../nyx_mode/packer/packer/nyx_config_gen.py nyx-test Kernel > /dev/null 2>&1 + + test -e nyx-test/config.ron && { + sudo modprobe -r kvm-intel + sudo modprobe -r kvm + sudo modprobe kvm enable_vmware_backdoor=y + sudo modprobe kvm-intel + #cat /sys/module/kvm/parameters/enable_vmware_backdoor + + mkdir -p in + echo 00000 > in/in + $ECHO "$GREY[*] running afl-fuzz for nyx_mode, this will take approx 10 seconds" + { + AFL_DEBUG=1 ../afl-fuzz -i in -o out -V05 -X -- ./nyx-test >>errors 2>&1 + } >>errors 2>&1 + test -n "$( ls out/default/queue/id:000002* 2>/dev/null )" && { + $ECHO "$GREEN[+] afl-fuzz is working correctly with nyx_mode" + RUNTIME=`grep execs_done out/default/fuzzer_stats | awk '{print$3}'` + rm -rf errors nyx-test test-instr in out + } || { + echo CUT------------------------------------------------------------------CUT + cat errors + echo CUT------------------------------------------------------------------CUT + $ECHO "$RED[!] afl-fuzz is not working correctly with nyx_mode" + CODE=1 + } + } || { + $ECHO "$RED[!] nyx_packer failed, likely install requirements not met." + CODE=1 + } + } || { + $ECHO "$RED[!] nyx_packer failed, likely install requirements not met." + CODE=1 + } + #rm -rf test-instr in out errors nyx-test + } + } || { + echo CUT------------------------------------------------------------------CUT + cat errors + echo CUT------------------------------------------------------------------CUT + $ECHO "$RED[!] afl-cc compilation of test targets failed - what is going on??" + CODE=1 + } +} || { + $ECHO "$YELLOW[-] nyx_mode is not compiled, cannot test" + INCOMPLETE=1 +} + +. ./test-post.sh diff --git a/test/test-pre.sh b/test/test-pre.sh index 1ca9dfb5..ce996415 100755 --- a/test/test-pre.sh +++ b/test/test-pre.sh @@ -20,7 +20,7 @@ echo foobar | grep -qE 'asd|oob' 2>/dev/null || { echo Error: grep command does test -e ./test-all.sh || cd $(dirname $0) || exit 1 test -e ./test-all.sh || { echo Error: you must be in the test/ directory ; exit 1 ; } export AFL_PATH=`pwd`/.. -export AFL_NO_AFFINITY=1 # workaround for travis that fails for no avail cores +export AFL_TRY_AFFINITY=1 # workaround for travis that fails for no avail cores echo 1 > test.1 echo 1 > test.2 diff --git a/utils/dynamic_covfilter/README.md b/utils/dynamic_covfilter/README.md new file mode 100644 index 00000000..381e0855 --- /dev/null +++ b/utils/dynamic_covfilter/README.md @@ -0,0 +1,60 @@ +# Dynamic Instrumentation Filter + +Sometimes it can be beneficial to limit the instrumentation feedback to +specific code locations. It is possible to do so at compile-time by simply +not instrumenting any undesired locations. However, there are situations +where doing this dynamically without requiring a new build can be beneficial. +Especially when dealing with larger builds, it is much more convenient to +select the target code locations at runtime instead of doing so at build time. + +There are two ways of doing this in AFL++. Both approaches require a build of +AFL++ with `CODE_COVERAGE=1`, so make sure to build AFL++ first by invoking + +`CODE_COVERAGE=1 make` + +Once you have built AFL++, you can choose out of two approaches: + +## Simple Selection with `AFL_PC_FILTER` + +This approach requires a build with `AFL_INSTRUMENTATION=llvmnative` or +`llvmcodecov` as well as an AddressSanitizer build with debug information. + +By setting the environment variable `AFL_PC_FILTER` to a string, the runtime +symbolizer is enabled in the AFL++ runtime. At startup, the runtime will call +the `__sanitizer_symbolize_pc` API to resolve every PC in every loaded module. +The runtime then matches the result using `strstr` and disables the PC guard +if the symbolized PC does not contain the specified string. + +This approach has the benefit of being very easy to use. The downside is that +it causes significant startup delays with large binaries and that it requires +an AddressSanitizer build. + +This method has no additional runtime overhead after startup. + +## Selection using pre-symbolized data file with `AFL_PC_FILTER_FILE` + +To avoid large startup time delays, a specific module can be pre-symbolized +using the `make_symbol_list.py` script. This script outputs a sorted list of +functions with their respective relative offsets and lengths in the target +binary: + +`python3 make_symbol_list.py libxul.so > libxul.symbols.txt` + +The resulting list can be filtered, e.g. using grep: + +`grep -i "webgl" libxul.symbols.txt > libxul.webgl.symbols.txt` + +Finally, you can run with `AFL_PC_FILTER_FILE=libxul.webgl.symbols.txt` to +restrict instrumentation feedback to the given locations. This approach only +has a minimal startup time delay due to the implementation only using binary +search on the given file per PC rather than reading debug information for every +PC. It also works well with Nyx, where symbolizing is usually disabled for the +target process to avoid delays with frequent crashes. + +Similar to the previous method, This approach requires a build with +`AFL_INSTRUMENTATION=llvmnative` or `llvmcodecov` as well debug information. +However, it does not require the ASan runtime as it doesn't do the symbolizing +in process. Due to the way it maps PCs to symbols, it is less accurate when it +comes to includes and inlines (it assumes all PCs within a function belong to +that function and originate from the same file). For most purposes, this should +be a reasonable simplification to quickly process even the largest binaries. diff --git a/utils/dynamic_covfilter/make_symbol_list.py b/utils/dynamic_covfilter/make_symbol_list.py new file mode 100644 index 00000000..d1dd6ab3 --- /dev/null +++ b/utils/dynamic_covfilter/make_symbol_list.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Written by Christian Holler + +import json +import os +import sys +import subprocess + +if len(sys.argv) != 2: + print("Usage: %s binfile" % os.path.basename(sys.argv[0])) + sys.exit(1) + +binfile = sys.argv[1] + +addr2len = {} +addrs = [] + +output = subprocess.check_output(["objdump", "-t", binfile]).decode("utf-8") +for line in output.splitlines(): + line = line.replace("\t", " ") + components = [x for x in line.split(" ") if x] + if not components: + continue + try: + start_addr = int(components[0], 16) + except ValueError: + continue + + # Length has variable position in objdump output + length = None + for comp in components[1:]: + if len(comp) == 16: + try: + length = int(comp, 16) + break + except: + continue + + if length is None: + print("ERROR: Couldn't determine function section length: %s" % line) + + func = components[-1] + + addrs.append(start_addr) + addr2len[str(hex(start_addr))] = str(length) + +# The search implementation in the AFL runtime expects everything sorted. +addrs.sort() +addrs = [str(hex(addr)) for addr in addrs] + +# We symbolize in one go to speed things up with large binaries. +output = subprocess.check_output([ + "llvm-addr2line", + "--output-style=JSON", + "-f", "-C", "-a", "-e", + binfile], + input="\n".join(addrs).encode("utf-8")).decode("utf-8") + +output = output.strip().splitlines() +for line in output: + output = json.loads(line) + if "Symbol" in output and output["Address"] in addr2len: + final_output = [ + output["Address"], + addr2len[output["Address"]], + os.path.basename(output["ModuleName"]), + output["Symbol"][0]["FileName"], + output["Symbol"][0]["FunctionName"] + ] + print("\t".join(final_output)) -- cgit 1.4.1 From ed1a6f8a570c6fcabee962f402d8d58f6cea77b7 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 3 Feb 2024 11:01:31 +0100 Subject: 2024 v4.10c release --- GNUmakefile.gcc_plugin | 2 +- README.md | 4 ++-- afl-cmin.bash | 2 +- afl-whatsup | 2 +- docs/Changelog.md | 3 +-- frida_mode/Scripting.md | 2 +- frida_mode/test/cmplog/cmplog.c | 2 +- frida_mode/test/deferred/testinstr.c | 2 +- frida_mode/test/dynamic/testinstr.c | 2 +- frida_mode/test/entry_point/testinstr.c | 2 +- frida_mode/test/exe/testinstr.c | 2 +- frida_mode/test/js/test.c | 2 +- frida_mode/test/js/test2.c | 2 +- frida_mode/test/output/testinstr.c | 2 +- frida_mode/test/perf/perf.c | 2 +- frida_mode/test/persistent_ret/testinstr.c | 2 +- frida_mode/test/testinstr/testinstr.c | 2 +- frida_mode/test/unstable/unstable.c | 2 +- frida_mode/util/frida_get_symbol_addr.sh | 2 +- include/afl-as.h | 2 +- include/afl-fuzz.h | 2 +- include/afl-prealloc.h | 2 +- include/alloc-inl.h | 2 +- include/cmplog.h | 2 +- include/common.h | 2 +- include/config.h | 4 ++-- include/debug.h | 2 +- include/forkserver.h | 2 +- include/hash.h | 2 +- include/list.h | 2 +- include/sharedmem.h | 2 +- include/snapshot-inl.h | 2 +- include/types.h | 2 +- include/xxhash.h | 2 +- instrumentation/afl-compiler-rt.o.c | 2 +- instrumentation/afl-gcc-cmplog-pass.so.cc | 2 +- instrumentation/afl-gcc-cmptrs-pass.so.cc | 2 +- instrumentation/afl-gcc-common.h | 2 +- instrumentation/afl-gcc-pass.so.cc | 2 +- instrumentation/afl-llvm-dict2file.so.cc | 2 +- instrumentation/afl-llvm-lto-instrumentlist.so.cc | 2 +- instrumentation/afl-llvm-pass.so.cc | 2 +- instrumentation/cmplog-instructions-pass.cc | 2 +- instrumentation/cmplog-routines-pass.cc | 2 +- instrumentation/cmplog-switches-pass.cc | 2 +- instrumentation/injection-pass.cc | 2 +- qemu_mode/build_qemu_support.sh | 2 +- qemu_mode/fastexit/Makefile | 2 +- qemu_mode/libcompcov/Makefile | 2 +- qemu_mode/libcompcov/compcovtest.cc | 2 +- qemu_mode/libcompcov/libcompcov.so.c | 2 +- qemu_mode/libqasan/Makefile | 2 +- qemu_mode/libqasan/hooks.c | 2 +- qemu_mode/libqasan/libqasan.c | 2 +- qemu_mode/libqasan/libqasan.h | 2 +- qemu_mode/libqasan/malloc.c | 2 +- qemu_mode/libqasan/patch.c | 2 +- qemu_mode/libqasan/string.c | 2 +- qemu_mode/libqasan/uninstrument.c | 2 +- qemu_mode/unsigaction/Makefile | 2 +- qemu_mode/util/qemu_get_symbol_addr.sh | 2 +- src/afl-analyze.c | 2 +- src/afl-as.c | 2 +- src/afl-cc.c | 2 +- src/afl-common.c | 2 +- src/afl-forkserver.c | 2 +- src/afl-fuzz-bitmap.c | 2 +- src/afl-fuzz-cmplog.c | 2 +- src/afl-fuzz-extras.c | 2 +- src/afl-fuzz-init.c | 2 +- src/afl-fuzz-mutators.c | 2 +- src/afl-fuzz-one.c | 2 +- src/afl-fuzz-python.c | 2 +- src/afl-fuzz-queue.c | 2 +- src/afl-fuzz-redqueen.c | 2 +- src/afl-fuzz-run.c | 2 +- src/afl-fuzz-state.c | 2 +- src/afl-fuzz-stats.c | 2 +- src/afl-fuzz.c | 2 +- src/afl-gotcpu.c | 2 +- src/afl-ld-lto.c | 2 +- src/afl-sharedmem.c | 2 +- src/afl-showmap.c | 2 +- src/afl-tmin.c | 2 +- test-instr.c | 2 +- unicorn_mode/build_unicorn_support.sh | 2 +- utils/afl_network_proxy/afl-network-client.c | 2 +- utils/afl_network_proxy/afl-network-server.c | 2 +- utils/afl_proxy/afl-proxy.c | 2 +- utils/afl_untracer/afl-untracer.c | 2 +- utils/afl_untracer/libtestinstr.c | 2 +- utils/argv_fuzzing/Makefile | 2 +- utils/argv_fuzzing/argvfuzz.c | 2 +- utils/distributed_fuzzing/sync_script.sh | 2 +- utils/libdislocator/libdislocator.so.c | 2 +- utils/libtokencap/libtokencap.so.c | 2 +- utils/persistent_mode/test-instr.c | 2 +- 97 files changed, 99 insertions(+), 100 deletions(-) (limited to 'utils') diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index 16c98399..8f06792d 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -11,7 +11,7 @@ # from Laszlo Szekeres. # # Copyright 2015 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index fd48cb14..f713e971 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ AFL++ logo -Release version: [4.09c](https://github.com/AFLplusplus/AFLplusplus/releases) +Release version: [4.10c](https://github.com/AFLplusplus/AFLplusplus/releases) -GitHub version: 4.10a +GitHub version: 4.10c Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) diff --git a/afl-cmin.bash b/afl-cmin.bash index fda48fb4..6c271220 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -7,7 +7,7 @@ # # Copyright 2014, 2015 Google Inc. All rights reserved. # -# Copyright 2019-2023 AFLplusplus +# Copyright 2019-2024 AFLplusplus # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/afl-whatsup b/afl-whatsup index 5b7cbcd6..aa081e41 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -6,7 +6,7 @@ # Originally written by Michal Zalewski # # Copyright 2015 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docs/Changelog.md b/docs/Changelog.md index 29081549..48003f4b 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -3,7 +3,7 @@ This is the list of all noteworthy changes made in every public release of the tool. See README.md for the general instruction manual. -### Version ++4.10a (dev) +### Version ++4.10c (release) - afl-fuzz: - default power schedule is now EXPLORE, due a fix in fast schedules explore is slightly better now. @@ -34,7 +34,6 @@ - updated the custom grammar mutator - document afl-cmin does not work on macOS (but afl-cmin.bash does) - ### Version ++4.09c (release) - afl-fuzz: - fixed the new mutation implementation for two bugs diff --git a/frida_mode/Scripting.md b/frida_mode/Scripting.md index dfd09e7b..653687f0 100644 --- a/frida_mode/Scripting.md +++ b/frida_mode/Scripting.md @@ -390,7 +390,7 @@ Consider the [following](test/js/test2.c) test code... -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/cmplog/cmplog.c b/frida_mode/test/cmplog/cmplog.c index 2565b35c..d397f36e 100644 --- a/frida_mode/test/cmplog/cmplog.c +++ b/frida_mode/test/cmplog/cmplog.c @@ -2,7 +2,7 @@ // // Author: Mateusz Jurczyk (mjurczyk@google.com) // -// Copyright 2019-2023 Google LLC +// Copyright 2019-2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/frida_mode/test/deferred/testinstr.c b/frida_mode/test/deferred/testinstr.c index 0ab44582..4e5124ed 100644 --- a/frida_mode/test/deferred/testinstr.c +++ b/frida_mode/test/deferred/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/dynamic/testinstr.c b/frida_mode/test/dynamic/testinstr.c index 8b285f6d..0abc61fd 100644 --- a/frida_mode/test/dynamic/testinstr.c +++ b/frida_mode/test/dynamic/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/entry_point/testinstr.c b/frida_mode/test/entry_point/testinstr.c index 24d9a615..75e71bda 100644 --- a/frida_mode/test/entry_point/testinstr.c +++ b/frida_mode/test/entry_point/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/exe/testinstr.c b/frida_mode/test/exe/testinstr.c index d965502e..7b603659 100644 --- a/frida_mode/test/exe/testinstr.c +++ b/frida_mode/test/exe/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/js/test.c b/frida_mode/test/js/test.c index 87c9cdf6..9799bf3b 100644 --- a/frida_mode/test/js/test.c +++ b/frida_mode/test/js/test.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/js/test2.c b/frida_mode/test/js/test2.c index 6b680a24..60b30eb5 100644 --- a/frida_mode/test/js/test2.c +++ b/frida_mode/test/js/test2.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/output/testinstr.c b/frida_mode/test/output/testinstr.c index d965502e..7b603659 100644 --- a/frida_mode/test/output/testinstr.c +++ b/frida_mode/test/output/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/perf/perf.c b/frida_mode/test/perf/perf.c index d9626974..55efba26 100644 --- a/frida_mode/test/perf/perf.c +++ b/frida_mode/test/perf/perf.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/persistent_ret/testinstr.c b/frida_mode/test/persistent_ret/testinstr.c index 12365ceb..85aa2b80 100644 --- a/frida_mode/test/persistent_ret/testinstr.c +++ b/frida_mode/test/persistent_ret/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/testinstr/testinstr.c b/frida_mode/test/testinstr/testinstr.c index d965502e..7b603659 100644 --- a/frida_mode/test/testinstr/testinstr.c +++ b/frida_mode/test/testinstr/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/unstable/unstable.c b/frida_mode/test/unstable/unstable.c index a87b6c74..16978e7e 100644 --- a/frida_mode/test/unstable/unstable.c +++ b/frida_mode/test/unstable/unstable.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/util/frida_get_symbol_addr.sh b/frida_mode/util/frida_get_symbol_addr.sh index 2e682255..53d5b802 100755 --- a/frida_mode/util/frida_get_symbol_addr.sh +++ b/frida_mode/util/frida_get_symbol_addr.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 AFLplusplus +# Copyright 2024 AFLplusplus # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/include/afl-as.h b/include/afl-as.h index 486314e2..612f34f4 100644 --- a/include/afl-as.h +++ b/include/afl-as.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index c2b09b2e..c24f39e2 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/afl-prealloc.h b/include/afl-prealloc.h index d19a7b52..3c621d79 100644 --- a/include/afl-prealloc.h +++ b/include/afl-prealloc.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/alloc-inl.h b/include/alloc-inl.h index cff808b2..0aa417be 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/cmplog.h b/include/cmplog.h index e4821444..6bfc146b 100644 --- a/include/cmplog.h +++ b/include/cmplog.h @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/common.h b/include/common.h index a9739a7d..0df07dee 100644 --- a/include/common.h +++ b/include/common.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/config.h b/include/config.h index 7ad73c2f..9349828f 100644 --- a/include/config.h +++ b/include/config.h @@ -10,7 +10,7 @@ Heiko Eissfeldt , Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.10a" +#define VERSION "++4.10c" /****************************************************** * * diff --git a/include/debug.h b/include/debug.h index 234d8fc4..4b812f8e 100644 --- a/include/debug.h +++ b/include/debug.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/forkserver.h b/include/forkserver.h index f1d3b5b1..be7f9e8d 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -12,7 +12,7 @@ Dominik Maier > Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/hash.h b/include/hash.h index 0243c5b7..5d56a108 100644 --- a/include/hash.h +++ b/include/hash.h @@ -15,7 +15,7 @@ Other code written by Michal Zalewski Copyright 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/list.h b/include/list.h index 283bf035..441eccd3 100644 --- a/include/list.h +++ b/include/list.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/sharedmem.h b/include/sharedmem.h index d32bd845..4484066e 100644 --- a/include/sharedmem.h +++ b/include/sharedmem.h @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/snapshot-inl.h b/include/snapshot-inl.h index 3864e473..b2c81402 100644 --- a/include/snapshot-inl.h +++ b/include/snapshot-inl.h @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/types.h b/include/types.h index d6476d82..22332135 100644 --- a/include/types.h +++ b/include/types.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/xxhash.h b/include/xxhash.h index a8bd6f27..9a880470 100644 --- a/include/xxhash.h +++ b/include/xxhash.h @@ -1,7 +1,7 @@ /* * xxHash - Extremely Fast Hash algorithm * Header File - * Copyright (C) 2012-2023 Yann Collet + * Copyright (C) 2012-2024 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 8e55d6a0..caa3c3a8 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -3,7 +3,7 @@ ------------------------------------------------ Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/afl-gcc-cmplog-pass.so.cc b/instrumentation/afl-gcc-cmplog-pass.so.cc index b4e6fda9..774dd5fd 100644 --- a/instrumentation/afl-gcc-cmplog-pass.so.cc +++ b/instrumentation/afl-gcc-cmplog-pass.so.cc @@ -3,7 +3,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. Copyright 2019-2020 AFLplusplus Project. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva , based on the AFL++ LLVM CmpLog pass by Andrea Fioraldi , and diff --git a/instrumentation/afl-gcc-cmptrs-pass.so.cc b/instrumentation/afl-gcc-cmptrs-pass.so.cc index c56263dd..929a9d7a 100644 --- a/instrumentation/afl-gcc-cmptrs-pass.so.cc +++ b/instrumentation/afl-gcc-cmptrs-pass.so.cc @@ -3,7 +3,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. Copyright 2019-2020 AFLplusplus Project. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva , based on the AFL++ LLVM CmpLog Routines pass by Andrea Fioraldi diff --git a/instrumentation/afl-gcc-common.h b/instrumentation/afl-gcc-common.h index 1d5eb466..80ded57d 100644 --- a/instrumentation/afl-gcc-common.h +++ b/instrumentation/afl-gcc-common.h @@ -2,7 +2,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva , based on the AFL++ GCC plugin. diff --git a/instrumentation/afl-gcc-pass.so.cc b/instrumentation/afl-gcc-pass.so.cc index 4d7fd0ef..41b1e5af 100644 --- a/instrumentation/afl-gcc-pass.so.cc +++ b/instrumentation/afl-gcc-pass.so.cc @@ -2,7 +2,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva , based on the AFL LLVM pass by Laszlo Szekeres and Michal diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index c60f3e06..ac497b5b 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -4,7 +4,7 @@ Written by Marc Heuse - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/afl-llvm-lto-instrumentlist.so.cc b/instrumentation/afl-llvm-lto-instrumentlist.so.cc index 61f97d77..e0899cd3 100644 --- a/instrumentation/afl-llvm-lto-instrumentlist.so.cc +++ b/instrumentation/afl-llvm-lto-instrumentlist.so.cc @@ -9,7 +9,7 @@ from afl-as.c are Michal's fault. Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index 052488a9..62f5023d 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -12,7 +12,7 @@ NGRAM previous location coverage comes from Adrian Herrera. Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index 8be8c294..dc60221e 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -5,7 +5,7 @@ Written by Andrea Fioraldi Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index b27e06e0..78317d5d 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -5,7 +5,7 @@ Written by Andrea Fioraldi Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/cmplog-switches-pass.cc b/instrumentation/cmplog-switches-pass.cc index 01da6da7..3e05c13d 100644 --- a/instrumentation/cmplog-switches-pass.cc +++ b/instrumentation/cmplog-switches-pass.cc @@ -5,7 +5,7 @@ Written by Andrea Fioraldi Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/injection-pass.cc b/instrumentation/injection-pass.cc index 971b103b..2280208b 100644 --- a/instrumentation/injection-pass.cc +++ b/instrumentation/injection-pass.cc @@ -5,7 +5,7 @@ Written by Marc Heuse Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index 3f8a88f2..45019cc8 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -13,7 +13,7 @@ # counters by Andrea Fioraldi # # Copyright 2015, 2016, 2017 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/fastexit/Makefile b/qemu_mode/fastexit/Makefile index c7b79277..be80207d 100644 --- a/qemu_mode/fastexit/Makefile +++ b/qemu_mode/fastexit/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/Makefile b/qemu_mode/libcompcov/Makefile index 7260df87..4761ac02 100644 --- a/qemu_mode/libcompcov/Makefile +++ b/qemu_mode/libcompcov/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/compcovtest.cc b/qemu_mode/libcompcov/compcovtest.cc index 23215013..11797091 100644 --- a/qemu_mode/libcompcov/compcovtest.cc +++ b/qemu_mode/libcompcov/compcovtest.cc @@ -2,7 +2,7 @@ // // Author: Mateusz Jurczyk (mjurczyk@google.com) // -// Copyright 2019-2023 Google LLC +// Copyright 2019-2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/libcompcov.so.c b/qemu_mode/libcompcov/libcompcov.so.c index b57e9701..36f7b2e2 100644 --- a/qemu_mode/libcompcov/libcompcov.so.c +++ b/qemu_mode/libcompcov/libcompcov.so.c @@ -5,7 +5,7 @@ Written and maintained by Andrea Fioraldi - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/qemu_mode/libqasan/Makefile b/qemu_mode/libqasan/Makefile index 61782894..7366d6f6 100644 --- a/qemu_mode/libqasan/Makefile +++ b/qemu_mode/libqasan/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libqasan/hooks.c b/qemu_mode/libqasan/hooks.c index a9fd0ce9..cf1b0820 100644 --- a/qemu_mode/libqasan/hooks.c +++ b/qemu_mode/libqasan/hooks.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/libqasan.c b/qemu_mode/libqasan/libqasan.c index 12be7778..45f47d5a 100644 --- a/qemu_mode/libqasan/libqasan.c +++ b/qemu_mode/libqasan/libqasan.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/libqasan.h b/qemu_mode/libqasan/libqasan.h index a430c868..f0844e23 100644 --- a/qemu_mode/libqasan/libqasan.h +++ b/qemu_mode/libqasan/libqasan.h @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/malloc.c b/qemu_mode/libqasan/malloc.c index 4448f480..ae470b56 100644 --- a/qemu_mode/libqasan/malloc.c +++ b/qemu_mode/libqasan/malloc.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/patch.c b/qemu_mode/libqasan/patch.c index 38e0903b..4ce8c3d8 100644 --- a/qemu_mode/libqasan/patch.c +++ b/qemu_mode/libqasan/patch.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/string.c b/qemu_mode/libqasan/string.c index e17cff4b..cd14d57b 100644 --- a/qemu_mode/libqasan/string.c +++ b/qemu_mode/libqasan/string.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/uninstrument.c b/qemu_mode/libqasan/uninstrument.c index e37a9b46..996f2a74 100644 --- a/qemu_mode/libqasan/uninstrument.c +++ b/qemu_mode/libqasan/uninstrument.c @@ -7,7 +7,7 @@ for some strange reason. */ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/unsigaction/Makefile b/qemu_mode/unsigaction/Makefile index c1a7397f..d5e807d8 100644 --- a/qemu_mode/unsigaction/Makefile +++ b/qemu_mode/unsigaction/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/util/qemu_get_symbol_addr.sh b/qemu_mode/util/qemu_get_symbol_addr.sh index e0a7ae80..5e00f1b2 100755 --- a/qemu_mode/util/qemu_get_symbol_addr.sh +++ b/qemu_mode/util/qemu_get_symbol_addr.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 AFLplusplus +# Copyright 2024 AFLplusplus # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 5b122741..95f32fee 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-as.c b/src/afl-as.c index 772e31b3..09ba75bf 100644 --- a/src/afl-as.c +++ b/src/afl-as.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-cc.c b/src/afl-cc.c index 98310545..e9564277 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -5,7 +5,7 @@ Written by Michal Zalewski, Laszlo Szekeres and Marc Heuse Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-common.c b/src/afl-common.c index ba498b3b..87003b03 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index ded0c21d..0a77d61c 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -13,7 +13,7 @@ Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 5f67347c..d056ac9f 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-cmplog.c b/src/afl-fuzz-cmplog.c index 3e6432ca..21f34e12 100644 --- a/src/afl-fuzz-cmplog.c +++ b/src/afl-fuzz-cmplog.c @@ -11,7 +11,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-extras.c b/src/afl-fuzz-extras.c index 905431d1..3b1d13f1 100644 --- a/src/afl-fuzz-extras.c +++ b/src/afl-fuzz-extras.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 057d8cf5..76291cc4 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 17fb9368..ae4d6668 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index c163a420..d9c074ec 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 4c7da774..16a398fd 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 67931bba..1ea50418 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 9e9b3822..eead7a8b 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -11,7 +11,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 1ee8ebe7..d764952c 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index b647ac84..4467cae8 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 4b83ad29..76577081 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 69064d51..12d67fe7 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-gotcpu.c b/src/afl-gotcpu.c index 4f851099..7aee2985 100644 --- a/src/afl-gotcpu.c +++ b/src/afl-gotcpu.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-ld-lto.c b/src/afl-ld-lto.c index 7ce5de41..513c1ae9 100644 --- a/src/afl-ld-lto.c +++ b/src/afl-ld-lto.c @@ -9,7 +9,7 @@ Andrea Fioraldi Dominik Maier - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-sharedmem.c b/src/afl-sharedmem.c index a2c81586..daea8f46 100644 --- a/src/afl-sharedmem.c +++ b/src/afl-sharedmem.c @@ -11,7 +11,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 7a639cf6..20ba5a5e 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-tmin.c b/src/afl-tmin.c index e7442d1d..4e5dab41 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/test-instr.c b/test-instr.c index eda5189c..28552893 100644 --- a/test-instr.c +++ b/test-instr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/unicorn_mode/build_unicorn_support.sh b/unicorn_mode/build_unicorn_support.sh index d3d16ad5..baca2171 100755 --- a/unicorn_mode/build_unicorn_support.sh +++ b/unicorn_mode/build_unicorn_support.sh @@ -14,7 +14,7 @@ # # # Copyright 2017 Battelle Memorial Institute. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/utils/afl_network_proxy/afl-network-client.c b/utils/afl_network_proxy/afl-network-client.c index 0416f0f9..1f04dd87 100644 --- a/utils/afl_network_proxy/afl-network-client.c +++ b/utils/afl_network_proxy/afl-network-client.c @@ -4,7 +4,7 @@ Written by Marc Heuse - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_network_proxy/afl-network-server.c b/utils/afl_network_proxy/afl-network-server.c index 95b0a551..c4a700f4 100644 --- a/utils/afl_network_proxy/afl-network-server.c +++ b/utils/afl_network_proxy/afl-network-server.c @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_proxy/afl-proxy.c b/utils/afl_proxy/afl-proxy.c index 531a97a2..6cf47636 100644 --- a/utils/afl_proxy/afl-proxy.c +++ b/utils/afl_proxy/afl-proxy.c @@ -4,7 +4,7 @@ Written by Marc Heuse - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_untracer/afl-untracer.c b/utils/afl_untracer/afl-untracer.c index 0e3f8a45..e6a74518 100644 --- a/utils/afl_untracer/afl-untracer.c +++ b/utils/afl_untracer/afl-untracer.c @@ -4,7 +4,7 @@ Written by Marc Heuse - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_untracer/libtestinstr.c b/utils/afl_untracer/libtestinstr.c index b7afc325..0a98778a 100644 --- a/utils/afl_untracer/libtestinstr.c +++ b/utils/afl_untracer/libtestinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/utils/argv_fuzzing/Makefile b/utils/argv_fuzzing/Makefile index 6786467a..ba977e5f 100644 --- a/utils/argv_fuzzing/Makefile +++ b/utils/argv_fuzzing/Makefile @@ -2,7 +2,7 @@ # american fuzzy lop++ - argvfuzz # -------------------------------- # -# Copyright 2019-2023 Kjell Braden +# Copyright 2019-2024 Kjell Braden # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/utils/argv_fuzzing/argvfuzz.c b/utils/argv_fuzzing/argvfuzz.c index 41eead0c..47383138 100644 --- a/utils/argv_fuzzing/argvfuzz.c +++ b/utils/argv_fuzzing/argvfuzz.c @@ -2,7 +2,7 @@ american fuzzy lop++ - LD_PRELOAD for fuzzing argv in binaries ------------------------------------------------------------ - Copyright 2019-2023 Kjell Braden + Copyright 2019-2024 Kjell Braden Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/distributed_fuzzing/sync_script.sh b/utils/distributed_fuzzing/sync_script.sh index b22816f1..861b65c8 100755 --- a/utils/distributed_fuzzing/sync_script.sh +++ b/utils/distributed_fuzzing/sync_script.sh @@ -6,7 +6,7 @@ # Originally written by Michal Zalewski # # Copyright 2014 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/utils/libdislocator/libdislocator.so.c b/utils/libdislocator/libdislocator.so.c index 1cd7abc6..b80be1a1 100644 --- a/utils/libdislocator/libdislocator.so.c +++ b/utils/libdislocator/libdislocator.so.c @@ -6,7 +6,7 @@ Originally written by Michal Zalewski Copyright 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/libtokencap/libtokencap.so.c b/utils/libtokencap/libtokencap.so.c index f4024799..cc499150 100644 --- a/utils/libtokencap/libtokencap.so.c +++ b/utils/libtokencap/libtokencap.so.c @@ -6,7 +6,7 @@ Originally written by Michal Zalewski Copyright 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/persistent_mode/test-instr.c b/utils/persistent_mode/test-instr.c index 4ead6577..72e26e93 100644 --- a/utils/persistent_mode/test-instr.c +++ b/utils/persistent_mode/test-instr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: -- cgit 1.4.1 From 602eceed8b56eef62d673a54b6011541bf1ab60a Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 3 Feb 2024 11:55:51 +0100 Subject: push to stable (#1983) * Output afl-clang-fast stuffs only if necessary (#1912) * afl-cc header * afl-cc common declarations - Add afl-cc-state.c - Strip includes, find_object, debug/be_quiet/have_*/callname setting from afl-cc.c - Use debugf_args in main - Modify execvp stuffs to fit new aflcc struct * afl-cc show usage * afl-cc mode selecting 1. compiler_mode by callname in argv[0] 2. compiler_mode by env "AFL_CC_COMPILER" 3. compiler_mode/instrument_mode by command line options "--afl-..." 4. instrument_mode/compiler_mode by various env vars including "AFL_LLVM_INSTRUMENT" 5. final checking steps 6. print "... - mode: %s-%s\n" 7. determine real argv[0] according to compiler_mode * afl-cc macro defs * afl-cc linking behaviors * afl-cc fsanitize behaviors * afl-cc misc * afl-cc body update * afl-cc all-in-one formated with custom-format.py * nits --------- Co-authored-by: vanhauser-thc * changelog * update grammar mutator * lto llvm 12+ * docs(custom_mutators): fix missing ':' (#1953) * Fix broken LTO mode and response file support (#1948) * Strip `-Wl,-no-undefined` during compilation (#1952) Make the compiler wrapper stripping `-Wl,-no-undefined` in addition to `-Wl,--no-undefined`. Both versions of the flag are accepted by clang and, therefore, used by building systems in the wild (e.g., samba will not build without this fix). * Remove dead code in write_to_testcase (#1955) The custom_mutators_count check in if case is duplicate with if condition. The else case is custom_mutators_count == 0, neither custom_mutator_list iteration nor sent check needed. Signed-off-by: Xeonacid * update qemuafl * WIP: Add ability to generate drcov trace using QEMU backend (#1956) * Document new drcov QEMU plugin * Add link to lightkeeper for QEMU drcov file loading --------- Co-authored-by: Jean-Romain Garnier * code format * changelog * sleep on uid != 0 afl-system-config * fix segv about skip_next, warn on unsupported cases of linking options (#1958) * todos * ensure afl-cc only allows available compiler modes * update grammar mutator * disable aslr on apple * fix for arm64 * help selective instrumentation * typos * macos * add compiler test script * apple fixes * bump nyx submodules (#1963) * fix docs * update changelog * update grammar mutator * improve compiler test script * gcc asan workaround (#1966) * fix github merge fuckup * fix * Fix afl-cc (#1968) - Check if too many cmdline params here, each time before insert a new param. - Check if it is "-fsanitize=..." before we do sth. - Remove improper param_st transfer. * Avoid adding llvmnative instrumentation when linking rust sanitizer runtime (#1969) * Dynamic instrumentation filtering for LLVM native (#1971) * Add two dynamic instrumentation filter methods to runtime * Always use pc-table with native pcguard * Add make_symbol_list.py and README * changelog * todos * new forkserver check * fix * nyx test for CI * improve nyx docs * Fixes to afl-cc and documentation (#1974) * Always compile with -ldl when building for CODE_COVERAGE When building with CODE_COVERAGE, the afl runtime contains code that calls `dladdr` which requires -ldl. Under most circumstances, clang already adds this (e.g. when building with pc-table), but there are some circumstances where it isn't added automatically. * Add visibility declaration to __afl_connected When building with hidden visibility, the use of __AFL_LOOP inside such code can cause linker errors due to __afl_connected being declared "hidden". * Update docs to clarify that CODE_COVERAGE=1 is required for dynamic_covfilter * nits * nyx build script updates * test error output * debug ci * debug ci * Improve afl-cc (#1975) * update response file support - full support of rsp file - fix some segv issues * Improve afl-cc - remove dead code about allow/denylist options of sancov - missing `if (!aflcc->have_msan)` - add docs for each function - typo * enable nyx * debug ci * debug ci * debug ci * debug ci * debug ci * debug ci * debug ci * debug ci * fix ci * clean test script * NO_NYX * NO_NYX * fix ci * debug ci * fix ci * finalize ci fix * Enhancement on Deterministic stage (#1972) * fuzzer: init commit based on aflpp 60dc37a8cf09f8e9048e4b6a2204d6c90b27655a * fuzzers: adding the skip variables and initialize * log: profile the det/havoc finding * log: add profile log output * fuzzers: sperate log/skipdet module * fuzzers: add quick eff_map calc * fuzzers: add skip_eff_map in fuzz_one * fuzzers: mark whole input space in eff_map * fuzzers: add undet bit threshold to skip some seeds * fuzzers: fix one byte overflow * fuzzers: fix overflow * fix code format * add havoc only again * code format * remove log to INTROSPECTION, rename skipdet module * rename skipdet module * remove log to stats * clean redundant code * code format * remove redundant code format check * remove redundant doc * remove redundant objects * clean files * change -d to default skipdet * disable deterministic when using CUSTOM_MUTATOR * revert fix * final touches for skipdet * remove unused var * remove redundant eff struct (#1977) * update QEMU-Nyx submodule (#1978) * update QEMU-Nyx submodule (#1980) * Fix type in AFL_NOOPT env variable in afl-cc help message (#1982) * nits * 2024 v4.10c release * fixes --------- Signed-off-by: Xeonacid Co-authored-by: Sonic <50692172+SonicStark@users.noreply.github.com> Co-authored-by: Xeonacid Co-authored-by: Nils Bars Co-authored-by: Jean-Romain Garnier <7504819+JRomainG@users.noreply.github.com> Co-authored-by: Jean-Romain Garnier Co-authored-by: Sergej Schumilo Co-authored-by: Christian Holler (:decoder) Co-authored-by: Han Zheng <35988108+kdsjZh@users.noreply.github.com> Co-authored-by: Khaled Yakdan --- .gitmodules | 6 +- GNUmakefile.gcc_plugin | 2 +- GNUmakefile.llvm | 2 +- README.md | 4 +- afl-cmin.bash | 2 +- afl-whatsup | 2 +- docs/Changelog.md | 5 +- docs/INSTALL.md | 4 +- frida_mode/Scripting.md | 2 +- frida_mode/test/cmplog/cmplog.c | 2 +- frida_mode/test/deferred/testinstr.c | 2 +- frida_mode/test/dynamic/testinstr.c | 2 +- frida_mode/test/entry_point/testinstr.c | 2 +- frida_mode/test/exe/testinstr.c | 2 +- frida_mode/test/js/test.c | 2 +- frida_mode/test/js/test2.c | 2 +- frida_mode/test/output/testinstr.c | 2 +- frida_mode/test/perf/perf.c | 2 +- frida_mode/test/persistent_ret/testinstr.c | 2 +- frida_mode/test/testinstr/testinstr.c | 2 +- frida_mode/test/unstable/unstable.c | 2 +- frida_mode/util/frida_get_symbol_addr.sh | 2 +- include/afl-as.h | 2 +- include/afl-fuzz.h | 60 +++- include/afl-prealloc.h | 2 +- include/alloc-inl.h | 2 +- include/cmplog.h | 2 +- include/common.h | 2 +- include/config.h | 16 +- include/debug.h | 2 +- include/forkserver.h | 5 +- include/hash.h | 2 +- include/list.h | 2 +- include/sharedmem.h | 2 +- include/snapshot-inl.h | 2 +- include/types.h | 2 +- include/xxhash.h | 2 +- instrumentation/afl-compiler-rt.o.c | 2 +- instrumentation/afl-gcc-cmplog-pass.so.cc | 2 +- instrumentation/afl-gcc-cmptrs-pass.so.cc | 2 +- instrumentation/afl-gcc-common.h | 2 +- instrumentation/afl-gcc-pass.so.cc | 2 +- instrumentation/afl-llvm-dict2file.so.cc | 2 +- instrumentation/afl-llvm-lto-instrumentlist.so.cc | 2 +- instrumentation/afl-llvm-pass.so.cc | 2 +- instrumentation/cmplog-instructions-pass.cc | 2 +- instrumentation/cmplog-routines-pass.cc | 2 +- instrumentation/cmplog-switches-pass.cc | 2 +- instrumentation/injection-pass.cc | 2 +- nyx_mode/QEMU-Nyx | 2 +- nyx_mode/QEMU_NYX_VERSION | 2 +- qemu_mode/build_qemu_support.sh | 2 +- qemu_mode/fastexit/Makefile | 2 +- qemu_mode/libcompcov/Makefile | 2 +- qemu_mode/libcompcov/compcovtest.cc | 2 +- qemu_mode/libcompcov/libcompcov.so.c | 2 +- qemu_mode/libqasan/Makefile | 2 +- qemu_mode/libqasan/hooks.c | 2 +- qemu_mode/libqasan/libqasan.c | 2 +- qemu_mode/libqasan/libqasan.h | 2 +- qemu_mode/libqasan/malloc.c | 2 +- qemu_mode/libqasan/patch.c | 2 +- qemu_mode/libqasan/string.c | 2 +- qemu_mode/libqasan/uninstrument.c | 2 +- qemu_mode/unsigaction/Makefile | 2 +- qemu_mode/util/qemu_get_symbol_addr.sh | 2 +- src/afl-analyze.c | 2 +- src/afl-as.c | 2 +- src/afl-cc.c | 4 +- src/afl-common.c | 2 +- src/afl-forkserver.c | 6 +- src/afl-fuzz-bitmap.c | 2 +- src/afl-fuzz-cmplog.c | 2 +- src/afl-fuzz-extras.c | 2 +- src/afl-fuzz-init.c | 17 +- src/afl-fuzz-mutators.c | 2 +- src/afl-fuzz-one.c | 237 ++++++------- src/afl-fuzz-python.c | 2 +- src/afl-fuzz-queue.c | 13 +- src/afl-fuzz-redqueen.c | 2 +- src/afl-fuzz-run.c | 2 +- src/afl-fuzz-skipdet.c | 403 ++++++++++++++++++++++ src/afl-fuzz-state.c | 10 +- src/afl-fuzz-stats.c | 40 ++- src/afl-fuzz.c | 27 +- src/afl-gotcpu.c | 2 +- src/afl-ld-lto.c | 2 +- src/afl-sharedmem.c | 2 +- src/afl-showmap.c | 2 +- src/afl-tmin.c | 2 +- test-instr.c | 2 +- test/test-custom-mutators.sh | 4 +- unicorn_mode/build_unicorn_support.sh | 2 +- utils/afl_network_proxy/afl-network-client.c | 2 +- utils/afl_network_proxy/afl-network-server.c | 2 +- utils/afl_proxy/afl-proxy.c | 2 +- utils/afl_untracer/afl-untracer.c | 2 +- utils/afl_untracer/libtestinstr.c | 2 +- utils/argv_fuzzing/Makefile | 2 +- utils/argv_fuzzing/argvfuzz.c | 2 +- utils/distributed_fuzzing/sync_script.sh | 2 +- utils/libdislocator/libdislocator.so.c | 2 +- utils/libtokencap/libtokencap.so.c | 2 +- utils/persistent_mode/test-instr.c | 2 +- 104 files changed, 793 insertions(+), 242 deletions(-) create mode 100644 src/afl-fuzz-skipdet.c (limited to 'utils') diff --git a/.gitmodules b/.gitmodules index 18fda27e..7fce4460 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,9 +19,9 @@ [submodule "nyx_mode/libnyx"] path = nyx_mode/libnyx url = https://github.com/nyx-fuzz/libnyx.git -[submodule "nyx_mode/QEMU-Nyx"] - path = nyx_mode/QEMU-Nyx - url = https://github.com/nyx-fuzz/qemu-nyx.git [submodule "nyx_mode/packer"] path = nyx_mode/packer url = https://github.com/nyx-fuzz/packer.git +[submodule "nyx_mode/QEMU-Nyx"] + path = nyx_mode/QEMU-Nyx + url = https://github.com/nyx-fuzz/QEMU-Nyx diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index 16c98399..8f06792d 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -11,7 +11,7 @@ # from Laszlo Szekeres. # # Copyright 2015 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index 7437130d..ec8fefe4 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -45,7 +45,7 @@ endif LLVMVER = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/git//' | sed 's/svn//' ) LLVM_MAJOR = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/\..*//' ) LLVM_MINOR = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/.*\.//' | sed 's/git//' | sed 's/svn//' | sed 's/ .*//' ) -LLVM_UNSUPPORTED = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^[0-2]\.|^3.[0-7]\.' && echo 1 || echo 0 ) +LLVM_UNSUPPORTED = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^[0-2]\.|^3.[0-8]\.' && echo 1 || echo 0 ) LLVM_TOO_NEW = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[8-9]|^2[0-9]' && echo 1 || echo 0 ) LLVM_TOO_OLD = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^[1-9]\.|^1[012]\.' && echo 1 || echo 0 ) LLVM_NEW_API = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[0-9]' && echo 1 || echo 0 ) diff --git a/README.md b/README.md index fd48cb14..f713e971 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ AFL++ logo -Release version: [4.09c](https://github.com/AFLplusplus/AFLplusplus/releases) +Release version: [4.10c](https://github.com/AFLplusplus/AFLplusplus/releases) -GitHub version: 4.10a +GitHub version: 4.10c Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) diff --git a/afl-cmin.bash b/afl-cmin.bash index fda48fb4..6c271220 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -7,7 +7,7 @@ # # Copyright 2014, 2015 Google Inc. All rights reserved. # -# Copyright 2019-2023 AFLplusplus +# Copyright 2019-2024 AFLplusplus # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/afl-whatsup b/afl-whatsup index 5b7cbcd6..aa081e41 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -6,7 +6,7 @@ # Originally written by Michal Zalewski # # Copyright 2015 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docs/Changelog.md b/docs/Changelog.md index 720a0689..48003f4b 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -3,12 +3,14 @@ This is the list of all noteworthy changes made in every public release of the tool. See README.md for the general instruction manual. -### Version ++4.10a (dev) +### Version ++4.10c (release) - afl-fuzz: - default power schedule is now EXPLORE, due a fix in fast schedules explore is slightly better now. - fixed minor issues in the mutation engine, thanks to @futhewo for reporting! + - better deterministic fuzzing is now available, benchmarks have shown + to improve fuzzing. Enable with -D. Thanks to @kdsjZh for the PR! - afl-cc: - large rewrite by @SonicStark which fixes a few corner cases, thanks! - LTO mode now requires llvm 12+ @@ -32,7 +34,6 @@ - updated the custom grammar mutator - document afl-cmin does not work on macOS (but afl-cmin.bash does) - ### Version ++4.09c (release) - afl-fuzz: - fixed the new mutation implementation for two bugs diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 1379df0a..84bbe3ea 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -114,10 +114,10 @@ freshly installed clang, clang++, llvm-config, gmake and coreutils, e.g.: # Depending on your MacOS system + brew version it is either export PATH="/opt/homebrew/opt/llvm/bin:$PATH" # or -export PATH="/usr/local/opt/llvm/bin:$PATH" +export PATH="/usr/local/opt/llvm/bin:/usr/local/opt/coreutils/libexec/gnubin:$PATH" # you can check with "brew info llvm" -export PATH="/usr/local/opt/coreutils/libexec/gnubin:/usr/local/bin:$PATH" +export PATH="/usr/local/bin:$PATH" export CC=clang export CXX=clang++ gmake diff --git a/frida_mode/Scripting.md b/frida_mode/Scripting.md index dfd09e7b..653687f0 100644 --- a/frida_mode/Scripting.md +++ b/frida_mode/Scripting.md @@ -390,7 +390,7 @@ Consider the [following](test/js/test2.c) test code... -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/cmplog/cmplog.c b/frida_mode/test/cmplog/cmplog.c index 2565b35c..d397f36e 100644 --- a/frida_mode/test/cmplog/cmplog.c +++ b/frida_mode/test/cmplog/cmplog.c @@ -2,7 +2,7 @@ // // Author: Mateusz Jurczyk (mjurczyk@google.com) // -// Copyright 2019-2023 Google LLC +// Copyright 2019-2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/frida_mode/test/deferred/testinstr.c b/frida_mode/test/deferred/testinstr.c index 0ab44582..4e5124ed 100644 --- a/frida_mode/test/deferred/testinstr.c +++ b/frida_mode/test/deferred/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/dynamic/testinstr.c b/frida_mode/test/dynamic/testinstr.c index 8b285f6d..0abc61fd 100644 --- a/frida_mode/test/dynamic/testinstr.c +++ b/frida_mode/test/dynamic/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/entry_point/testinstr.c b/frida_mode/test/entry_point/testinstr.c index 24d9a615..75e71bda 100644 --- a/frida_mode/test/entry_point/testinstr.c +++ b/frida_mode/test/entry_point/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/exe/testinstr.c b/frida_mode/test/exe/testinstr.c index d965502e..7b603659 100644 --- a/frida_mode/test/exe/testinstr.c +++ b/frida_mode/test/exe/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/js/test.c b/frida_mode/test/js/test.c index 87c9cdf6..9799bf3b 100644 --- a/frida_mode/test/js/test.c +++ b/frida_mode/test/js/test.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/js/test2.c b/frida_mode/test/js/test2.c index 6b680a24..60b30eb5 100644 --- a/frida_mode/test/js/test2.c +++ b/frida_mode/test/js/test2.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/output/testinstr.c b/frida_mode/test/output/testinstr.c index d965502e..7b603659 100644 --- a/frida_mode/test/output/testinstr.c +++ b/frida_mode/test/output/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/perf/perf.c b/frida_mode/test/perf/perf.c index d9626974..55efba26 100644 --- a/frida_mode/test/perf/perf.c +++ b/frida_mode/test/perf/perf.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/persistent_ret/testinstr.c b/frida_mode/test/persistent_ret/testinstr.c index 12365ceb..85aa2b80 100644 --- a/frida_mode/test/persistent_ret/testinstr.c +++ b/frida_mode/test/persistent_ret/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/testinstr/testinstr.c b/frida_mode/test/testinstr/testinstr.c index d965502e..7b603659 100644 --- a/frida_mode/test/testinstr/testinstr.c +++ b/frida_mode/test/testinstr/testinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/test/unstable/unstable.c b/frida_mode/test/unstable/unstable.c index a87b6c74..16978e7e 100644 --- a/frida_mode/test/unstable/unstable.c +++ b/frida_mode/test/unstable/unstable.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/frida_mode/util/frida_get_symbol_addr.sh b/frida_mode/util/frida_get_symbol_addr.sh index 2e682255..53d5b802 100755 --- a/frida_mode/util/frida_get_symbol_addr.sh +++ b/frida_mode/util/frida_get_symbol_addr.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 AFLplusplus +# Copyright 2024 AFLplusplus # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/include/afl-as.h b/include/afl-as.h index 486314e2..612f34f4 100644 --- a/include/afl-as.h +++ b/include/afl-as.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index f1813df6..c24f39e2 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -149,6 +149,48 @@ struct tainted { }; +struct inf_profile { + + u32 inf_skipped_bytes; /* Inference Stage Profiling */ + u64 inf_execs_cost, inf_time_cost; + +}; + +/* ToDo: add cmplog profile as well */ +struct havoc_profile { + + u32 queued_det_stage, /* Det/Havoc Stage Profiling */ + queued_havoc_stage, total_queued_det, edge_det_stage, edge_havoc_stage, + total_det_edge; + + u64 det_stage_time, havoc_stage_time, total_det_time; + +}; + +struct skipdet_entry { + + u8 continue_inf, done_eff; + u32 undet_bits, quick_eff_bytes; + + u8 *skip_eff_map, /* we'v finish the eff_map */ + *done_inf_map; /* some bytes are not done yet */ + +}; + +struct skipdet_global { + + u8 use_skip_havoc; + + u32 undet_bits_threshold; + + u64 last_cov_undet; + + u8 *virgin_det_bits; /* global fuzzed bits */ + + struct inf_profile *inf_prof; + +}; + struct queue_entry { u8 *fname; /* File name for the test case */ @@ -203,6 +245,8 @@ struct queue_entry { struct queue_entry *mother; /* queue entry this based on */ + struct skipdet_entry *skipdet_e; + }; struct extra_data { @@ -247,6 +291,8 @@ enum { /* 19 */ STAGE_CUSTOM_MUTATOR, /* 20 */ STAGE_COLORIZATION, /* 21 */ STAGE_ITS, + /* 22 */ STAGE_INF, + /* 23 */ STAGE_QUICK, STAGE_NUM_MAX @@ -782,6 +828,11 @@ typedef struct afl_state { * is too large) */ struct queue_entry **q_testcase_cache; + /* Global Profile Data for deterministic/havoc-splice stage */ + struct havoc_profile *havoc_prof; + + struct skipdet_global *skipdet_g; + #ifdef INTROSPECTION char mutation[8072]; char m_tmp[4096]; @@ -1232,6 +1283,13 @@ AFL_RAND_RETURN rand_next(afl_state_t *afl); /* probability between 0.0 and 1.0 */ double rand_next_percent(afl_state_t *afl); +/* SkipDet Functions */ + +u8 skip_deterministic_stage(afl_state_t *, u8 *, u8 *, u32, u64); +u8 is_det_timeout(u64, u8); + +void plot_profile_data(afl_state_t *, struct queue_entry *); + /**** Inline routines ****/ /* Generate a random number (from 0 to limit - 1). This may diff --git a/include/afl-prealloc.h b/include/afl-prealloc.h index d19a7b52..3c621d79 100644 --- a/include/afl-prealloc.h +++ b/include/afl-prealloc.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/alloc-inl.h b/include/alloc-inl.h index cff808b2..0aa417be 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/cmplog.h b/include/cmplog.h index e4821444..6bfc146b 100644 --- a/include/cmplog.h +++ b/include/cmplog.h @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/common.h b/include/common.h index a9739a7d..0df07dee 100644 --- a/include/common.h +++ b/include/common.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/config.h b/include/config.h index 63340650..9349828f 100644 --- a/include/config.h +++ b/include/config.h @@ -10,7 +10,7 @@ Heiko Eissfeldt , Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.10a" +#define VERSION "++4.10c" /****************************************************** * * @@ -52,6 +52,18 @@ /* Default file permission umode when creating files (default: 0600) */ #define DEFAULT_PERMISSION 0600 +/* SkipDet's global configuration */ + +#define MINIMAL_BLOCK_SIZE 64 +#define SMALL_DET_TIME (60 * 1000 * 1000U) +#define MAXIMUM_INF_EXECS (16 * 1024U) +#define MAXIMUM_QUICK_EFF_EXECS (64 * 1024U) +#define THRESHOLD_DEC_TIME (20 * 60 * 1000U) + +/* Set the Prob of selecting eff_bytes 3 times more than original, + Now disabled */ +#define EFF_HAVOC_RATE 3 + /* CMPLOG/REDQUEEN TUNING * * Here you can modify tuning and solving options for CMPLOG. diff --git a/include/debug.h b/include/debug.h index 234d8fc4..4b812f8e 100644 --- a/include/debug.h +++ b/include/debug.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/forkserver.h b/include/forkserver.h index f6230fe8..be7f9e8d 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -12,7 +12,7 @@ Dominik Maier > Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -126,7 +126,8 @@ typedef struct afl_forkserver { u8 *out_file, /* File to fuzz, if any */ *target_path; /* Path of the target */ - FILE *plot_file; /* Gnuplot output file */ + FILE *plot_file, /* Gnuplot output file */ + *det_plot_file; /* Note: last_run_timed_out is u32 to send it to the child as 4 byte array */ u32 last_run_timed_out; /* Traced process timed out? */ diff --git a/include/hash.h b/include/hash.h index 0243c5b7..5d56a108 100644 --- a/include/hash.h +++ b/include/hash.h @@ -15,7 +15,7 @@ Other code written by Michal Zalewski Copyright 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/list.h b/include/list.h index 283bf035..441eccd3 100644 --- a/include/list.h +++ b/include/list.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/sharedmem.h b/include/sharedmem.h index d32bd845..4484066e 100644 --- a/include/sharedmem.h +++ b/include/sharedmem.h @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/snapshot-inl.h b/include/snapshot-inl.h index 3864e473..b2c81402 100644 --- a/include/snapshot-inl.h +++ b/include/snapshot-inl.h @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/types.h b/include/types.h index d6476d82..22332135 100644 --- a/include/types.h +++ b/include/types.h @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/include/xxhash.h b/include/xxhash.h index a8bd6f27..9a880470 100644 --- a/include/xxhash.h +++ b/include/xxhash.h @@ -1,7 +1,7 @@ /* * xxHash - Extremely Fast Hash algorithm * Header File - * Copyright (C) 2012-2023 Yann Collet + * Copyright (C) 2012-2024 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 8e55d6a0..caa3c3a8 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -3,7 +3,7 @@ ------------------------------------------------ Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/afl-gcc-cmplog-pass.so.cc b/instrumentation/afl-gcc-cmplog-pass.so.cc index b4e6fda9..774dd5fd 100644 --- a/instrumentation/afl-gcc-cmplog-pass.so.cc +++ b/instrumentation/afl-gcc-cmplog-pass.so.cc @@ -3,7 +3,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. Copyright 2019-2020 AFLplusplus Project. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva , based on the AFL++ LLVM CmpLog pass by Andrea Fioraldi , and diff --git a/instrumentation/afl-gcc-cmptrs-pass.so.cc b/instrumentation/afl-gcc-cmptrs-pass.so.cc index c56263dd..929a9d7a 100644 --- a/instrumentation/afl-gcc-cmptrs-pass.so.cc +++ b/instrumentation/afl-gcc-cmptrs-pass.so.cc @@ -3,7 +3,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. Copyright 2019-2020 AFLplusplus Project. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva , based on the AFL++ LLVM CmpLog Routines pass by Andrea Fioraldi diff --git a/instrumentation/afl-gcc-common.h b/instrumentation/afl-gcc-common.h index 1d5eb466..80ded57d 100644 --- a/instrumentation/afl-gcc-common.h +++ b/instrumentation/afl-gcc-common.h @@ -2,7 +2,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva , based on the AFL++ GCC plugin. diff --git a/instrumentation/afl-gcc-pass.so.cc b/instrumentation/afl-gcc-pass.so.cc index 4d7fd0ef..41b1e5af 100644 --- a/instrumentation/afl-gcc-pass.so.cc +++ b/instrumentation/afl-gcc-pass.so.cc @@ -2,7 +2,7 @@ Copyright 2014-2019 Free Software Foundation, Inc Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AdaCore + Copyright 2019-2024 AdaCore Written by Alexandre Oliva , based on the AFL LLVM pass by Laszlo Szekeres and Michal diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index c60f3e06..ac497b5b 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -4,7 +4,7 @@ Written by Marc Heuse - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/afl-llvm-lto-instrumentlist.so.cc b/instrumentation/afl-llvm-lto-instrumentlist.so.cc index 61f97d77..e0899cd3 100644 --- a/instrumentation/afl-llvm-lto-instrumentlist.so.cc +++ b/instrumentation/afl-llvm-lto-instrumentlist.so.cc @@ -9,7 +9,7 @@ from afl-as.c are Michal's fault. Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index 052488a9..62f5023d 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -12,7 +12,7 @@ NGRAM previous location coverage comes from Adrian Herrera. Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index 8be8c294..dc60221e 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -5,7 +5,7 @@ Written by Andrea Fioraldi Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index b27e06e0..78317d5d 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -5,7 +5,7 @@ Written by Andrea Fioraldi Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/cmplog-switches-pass.cc b/instrumentation/cmplog-switches-pass.cc index 01da6da7..3e05c13d 100644 --- a/instrumentation/cmplog-switches-pass.cc +++ b/instrumentation/cmplog-switches-pass.cc @@ -5,7 +5,7 @@ Written by Andrea Fioraldi Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/instrumentation/injection-pass.cc b/instrumentation/injection-pass.cc index 971b103b..2280208b 100644 --- a/instrumentation/injection-pass.cc +++ b/instrumentation/injection-pass.cc @@ -5,7 +5,7 @@ Written by Marc Heuse Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/nyx_mode/QEMU-Nyx b/nyx_mode/QEMU-Nyx index 1def26f8..e5e1c4c2 160000 --- a/nyx_mode/QEMU-Nyx +++ b/nyx_mode/QEMU-Nyx @@ -1 +1 @@ -Subproject commit 1def26f83e83556d767754581fa52081ffb54b09 +Subproject commit e5e1c4c21ff9c4dc80e6409d4eab47146c6024cd diff --git a/nyx_mode/QEMU_NYX_VERSION b/nyx_mode/QEMU_NYX_VERSION index cac32d41..c6ed0c6a 100644 --- a/nyx_mode/QEMU_NYX_VERSION +++ b/nyx_mode/QEMU_NYX_VERSION @@ -1 +1 @@ -1def26f83e +e5e1c4c21ff9c4dc80e6409d4eab47146c6024cd diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index 3f8a88f2..45019cc8 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -13,7 +13,7 @@ # counters by Andrea Fioraldi # # Copyright 2015, 2016, 2017 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/fastexit/Makefile b/qemu_mode/fastexit/Makefile index c7b79277..be80207d 100644 --- a/qemu_mode/fastexit/Makefile +++ b/qemu_mode/fastexit/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/Makefile b/qemu_mode/libcompcov/Makefile index 7260df87..4761ac02 100644 --- a/qemu_mode/libcompcov/Makefile +++ b/qemu_mode/libcompcov/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/compcovtest.cc b/qemu_mode/libcompcov/compcovtest.cc index 23215013..11797091 100644 --- a/qemu_mode/libcompcov/compcovtest.cc +++ b/qemu_mode/libcompcov/compcovtest.cc @@ -2,7 +2,7 @@ // // Author: Mateusz Jurczyk (mjurczyk@google.com) // -// Copyright 2019-2023 Google LLC +// Copyright 2019-2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/libcompcov.so.c b/qemu_mode/libcompcov/libcompcov.so.c index b57e9701..36f7b2e2 100644 --- a/qemu_mode/libcompcov/libcompcov.so.c +++ b/qemu_mode/libcompcov/libcompcov.so.c @@ -5,7 +5,7 @@ Written and maintained by Andrea Fioraldi - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/qemu_mode/libqasan/Makefile b/qemu_mode/libqasan/Makefile index 61782894..7366d6f6 100644 --- a/qemu_mode/libqasan/Makefile +++ b/qemu_mode/libqasan/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libqasan/hooks.c b/qemu_mode/libqasan/hooks.c index a9fd0ce9..cf1b0820 100644 --- a/qemu_mode/libqasan/hooks.c +++ b/qemu_mode/libqasan/hooks.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/libqasan.c b/qemu_mode/libqasan/libqasan.c index 12be7778..45f47d5a 100644 --- a/qemu_mode/libqasan/libqasan.c +++ b/qemu_mode/libqasan/libqasan.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/libqasan.h b/qemu_mode/libqasan/libqasan.h index a430c868..f0844e23 100644 --- a/qemu_mode/libqasan/libqasan.h +++ b/qemu_mode/libqasan/libqasan.h @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/malloc.c b/qemu_mode/libqasan/malloc.c index 4448f480..ae470b56 100644 --- a/qemu_mode/libqasan/malloc.c +++ b/qemu_mode/libqasan/malloc.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/patch.c b/qemu_mode/libqasan/patch.c index 38e0903b..4ce8c3d8 100644 --- a/qemu_mode/libqasan/patch.c +++ b/qemu_mode/libqasan/patch.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/string.c b/qemu_mode/libqasan/string.c index e17cff4b..cd14d57b 100644 --- a/qemu_mode/libqasan/string.c +++ b/qemu_mode/libqasan/string.c @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/libqasan/uninstrument.c b/qemu_mode/libqasan/uninstrument.c index e37a9b46..996f2a74 100644 --- a/qemu_mode/libqasan/uninstrument.c +++ b/qemu_mode/libqasan/uninstrument.c @@ -7,7 +7,7 @@ for some strange reason. */ /******************************************************************************* -Copyright (c) 2019-2023, Andrea Fioraldi +Copyright (c) 2019-2024, Andrea Fioraldi Redistribution and use in source and binary forms, with or without diff --git a/qemu_mode/unsigaction/Makefile b/qemu_mode/unsigaction/Makefile index c1a7397f..d5e807d8 100644 --- a/qemu_mode/unsigaction/Makefile +++ b/qemu_mode/unsigaction/Makefile @@ -4,7 +4,7 @@ # # Written by Andrea Fioraldi # -# Copyright 2019-2023 Andrea Fioraldi. All rights reserved. +# Copyright 2019-2024 Andrea Fioraldi. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/util/qemu_get_symbol_addr.sh b/qemu_mode/util/qemu_get_symbol_addr.sh index e0a7ae80..5e00f1b2 100755 --- a/qemu_mode/util/qemu_get_symbol_addr.sh +++ b/qemu_mode/util/qemu_get_symbol_addr.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 AFLplusplus +# Copyright 2024 AFLplusplus # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 5b122741..95f32fee 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-as.c b/src/afl-as.c index 772e31b3..09ba75bf 100644 --- a/src/afl-as.c +++ b/src/afl-as.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-cc.c b/src/afl-cc.c index c300ddfc..e9564277 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -5,7 +5,7 @@ Written by Michal Zalewski, Laszlo Szekeres and Marc Heuse Copyright 2015, 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2830,7 +2830,7 @@ static void maybe_usage(aflcc_state_t *aflcc, int argc, char **argv) { " AFL_DONT_OPTIMIZE: disable optimization instead of -O3\n" " AFL_NO_BUILTIN: no builtins for string compare functions (for " "libtokencap.so)\n" - " AFL_NOOP: behave like a normal compiler (to pass configure " + " AFL_NOOPT: behave like a normal compiler (to pass configure " "tests)\n" " AFL_PATH: path to instrumenting pass and runtime " "(afl-compiler-rt.*o)\n" diff --git a/src/afl-common.c b/src/afl-common.c index ba498b3b..87003b03 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 214b4fe9..0a77d61c 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -13,7 +13,7 @@ Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1019,7 +1019,9 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (status >= 0x41464c00 && status <= 0x41464cff) { - FATAL("Target uses the new forkserver model, you need to switch to a newer afl-fuzz too!"); + FATAL( + "Target uses the new forkserver model, you need to switch to a newer " + "afl-fuzz too!"); } diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 5f67347c..d056ac9f 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-cmplog.c b/src/afl-fuzz-cmplog.c index 3e6432ca..21f34e12 100644 --- a/src/afl-fuzz-cmplog.c +++ b/src/afl-fuzz-cmplog.c @@ -11,7 +11,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-extras.c b/src/afl-fuzz-extras.c index 905431d1..3b1d13f1 100644 --- a/src/afl-fuzz-extras.c +++ b/src/afl-fuzz-extras.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 8ab44a3b..76291cc4 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2236,6 +2236,21 @@ void setup_dirs_fds(afl_state_t *afl) { fflush(afl->fsrv.plot_file); +#ifdef INTROSPECTION + + tmp = alloc_printf("%s/plot_det_data", afl->out_dir); + + int fd = open(tmp, O_WRONLY | O_CREAT, DEFAULT_PERMISSION); + if (fd < 0) { PFATAL("Unable to create '%s'", tmp); } + ck_free(tmp); + + afl->fsrv.det_plot_file = fdopen(fd, "w"); + if (!afl->fsrv.det_plot_file) { PFATAL("fdopen() failed"); } + + if (afl->in_place_resume) { fseek(afl->fsrv.det_plot_file, 0, SEEK_END); } + +#endif + /* ignore errors */ } diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 17fb9368..ae4d6668 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 01e34b69..d9c074ec 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -329,9 +329,9 @@ u8 fuzz_one_original(afl_state_t *afl) { u32 len, temp_len; u32 j; u32 i; - u8 *in_buf, *out_buf, *orig_in, *ex_tmp, *eff_map = 0; + u8 *in_buf, *out_buf, *orig_in, *ex_tmp; u64 havoc_queued = 0, orig_hit_cnt, new_hit_cnt = 0, prev_cksum, _prev_cksum; - u32 splice_cycle = 0, perf_score = 100, orig_perf, eff_cnt = 1; + u32 splice_cycle = 0, perf_score = 100, orig_perf; u8 ret_val = 1, doing_det = 0; @@ -545,12 +545,37 @@ u8 fuzz_one_original(afl_state_t *afl) { } + u64 before_det_time = get_cur_time(); +#ifdef INTROSPECTION + + u64 before_havoc_time; + u32 before_det_findings = afl->queued_items, + before_det_edges = count_non_255_bytes(afl, afl->virgin_bits), + before_havoc_findings, before_havoc_edges; + u8 is_logged = 0; + +#endif + if (!afl->skip_deterministic) { + + if (!skip_deterministic_stage(afl, in_buf, out_buf, len, before_det_time)) { + + goto abandon_entry; + + } + + } + + u8 *skip_eff_map = afl->queue_cur->skipdet_e->skip_eff_map; + /* Skip right away if -d is given, if it has not been chosen sufficiently often to warrant the expensive deterministic stage (fuzz_level), or if it has gone through deterministic testing in earlier, resumed runs (passed_det). */ + /* if skipdet decide to skip the seed or no interesting bytes found, + we skip the whole deterministic stage as well */ if (likely(afl->skip_deterministic) || likely(afl->queue_cur->passed_det) || + likely(!afl->queue_cur->skipdet_e->quick_eff_bytes) || likely(perf_score < (afl->queue_cur->depth * 30 <= afl->havoc_max_mult * 100 ? afl->queue_cur->depth * 30 @@ -609,6 +634,10 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->stage_cur_byte = afl->stage_cur >> 3; + if (!skip_eff_map[afl->stage_cur_byte]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + FLIP_BIT(out_buf, afl->stage_cur); #ifdef INTROSPECTION @@ -725,6 +754,10 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->stage_cur_byte = afl->stage_cur >> 3; + if (!skip_eff_map[afl->stage_cur_byte]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + FLIP_BIT(out_buf, afl->stage_cur); FLIP_BIT(out_buf, afl->stage_cur + 1); @@ -760,6 +793,10 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->stage_cur_byte = afl->stage_cur >> 3; + if (!skip_eff_map[afl->stage_cur_byte]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + FLIP_BIT(out_buf, afl->stage_cur); FLIP_BIT(out_buf, afl->stage_cur + 1); FLIP_BIT(out_buf, afl->stage_cur + 2); @@ -787,34 +824,6 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->queue_cur->stats_mutated += afl->stage_max; #endif - /* Effector map setup. These macros calculate: - - EFF_APOS - position of a particular file offset in the map. - EFF_ALEN - length of a map with a particular number of bytes. - EFF_SPAN_ALEN - map span for a sequence of bytes. - - */ - -#define EFF_APOS(_p) ((_p) >> EFF_MAP_SCALE2) -#define EFF_REM(_x) ((_x) & ((1 << EFF_MAP_SCALE2) - 1)) -#define EFF_ALEN(_l) (EFF_APOS(_l) + !!EFF_REM(_l)) -#define EFF_SPAN_ALEN(_p, _l) (EFF_APOS((_p) + (_l)-1) - EFF_APOS(_p) + 1) - - /* Initialize effector map for the next step (see comments below). Always - flag first and last byte as doing something. */ - - eff_map = afl_realloc(AFL_BUF_PARAM(eff), EFF_ALEN(len)); - if (unlikely(!eff_map)) { PFATAL("alloc"); } - memset(eff_map, 0, EFF_ALEN(len)); - eff_map[0] = 1; - - if (EFF_APOS(len - 1) != 0) { - - eff_map[EFF_APOS(len - 1)] = 1; - ++eff_cnt; - - } - /* Walking byte. */ afl->stage_name = "bitflip 8/8"; @@ -828,6 +837,10 @@ u8 fuzz_one_original(afl_state_t *afl) { afl->stage_cur_byte = afl->stage_cur; + if (!skip_eff_map[afl->stage_cur_byte]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + out_buf[afl->stage_cur] ^= 0xFF; #ifdef INTROSPECTION @@ -837,59 +850,19 @@ u8 fuzz_one_original(afl_state_t *afl) { if (common_fuzz_stuff(afl, out_buf, len)) { goto abandon_entry; } - /* We also use this stage to pull off a simple trick: we identify - bytes that seem to have no effect on the current execution path - even when fully flipped - and we skip them during more expensive - deterministic stages, such as arithmetics or known ints. */ - - if (!eff_map[EFF_APOS(afl->stage_cur)]) { - - u64 cksum; - - /* If in non-instrumented mode or if the file is very short, just flag - everything without wasting time on checksums. */ - - if (!afl->non_instrumented_mode && len >= EFF_MIN_LEN) { - - cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); - - } else { - - cksum = ~prev_cksum; - - } - - if (cksum != prev_cksum) { - - eff_map[EFF_APOS(afl->stage_cur)] = 1; - ++eff_cnt; - - } - - } - out_buf[afl->stage_cur] ^= 0xFF; } - /* If the effector map is more than EFF_MAX_PERC dense, just flag the - whole thing as worth fuzzing, since we wouldn't be saving much time - anyway. */ - - if (eff_cnt != (u32)EFF_ALEN(len) && - eff_cnt * 100 / EFF_ALEN(len) > EFF_MAX_PERC) { + /* New effective bytes calculation. */ - memset(eff_map, 1, EFF_ALEN(len)); + for (i = 0; i < len; i++) { - afl->blocks_eff_select += EFF_ALEN(len); - - } else { - - afl->blocks_eff_select += eff_cnt; + if (skip_eff_map[i]) afl->blocks_eff_select += 1; } - afl->blocks_eff_total += EFF_ALEN(len); + afl->blocks_eff_total += len; new_hit_cnt = afl->queued_items + afl->saved_crashes; @@ -914,12 +887,9 @@ u8 fuzz_one_original(afl_state_t *afl) { /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)]) { - - --afl->stage_max; - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -959,13 +929,10 @@ u8 fuzz_one_original(afl_state_t *afl) { for (i = 0; i < len - 3; ++i) { /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)] && - !eff_map[EFF_APOS(i + 2)] && !eff_map[EFF_APOS(i + 3)]) { - --afl->stage_max; - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1016,12 +983,9 @@ skip_bitflip: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)]) { - - afl->stage_max -= 2 * ARITH_MAX; - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1103,12 +1067,9 @@ skip_bitflip: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)]) { - - afl->stage_max -= 4 * ARITH_MAX; - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1236,13 +1197,9 @@ skip_bitflip: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)] && - !eff_map[EFF_APOS(i + 2)] && !eff_map[EFF_APOS(i + 3)]) { - - afl->stage_max -= 4 * ARITH_MAX; - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1374,12 +1331,9 @@ skip_arith: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)]) { - - afl->stage_max -= sizeof(interesting_8); - continue; + if (!skip_eff_map[i]) continue; - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1437,12 +1391,9 @@ skip_arith: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)]) { + if (!skip_eff_map[i]) continue; - afl->stage_max -= sizeof(interesting_16); - continue; - - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1528,13 +1479,9 @@ skip_arith: /* Let's consult the effector map... */ - if (!eff_map[EFF_APOS(i)] && !eff_map[EFF_APOS(i + 1)] && - !eff_map[EFF_APOS(i + 2)] && !eff_map[EFF_APOS(i + 3)]) { + if (!skip_eff_map[i]) continue; - afl->stage_max -= sizeof(interesting_32) >> 1; - continue; - - } + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } afl->stage_cur_byte = i; @@ -1626,6 +1573,10 @@ skip_interest: u32 last_len = 0; + if (!skip_eff_map[i]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + afl->stage_cur_byte = i; /* Extras are sorted by size, from smallest to largest. This means @@ -1643,9 +1594,7 @@ skip_interest: if ((afl->extras_cnt > afl->max_det_extras && rand_below(afl, afl->extras_cnt) >= afl->max_det_extras) || afl->extras[j].len > len - i || - !memcmp(afl->extras[j].data, out_buf + i, afl->extras[j].len) || - !memchr(eff_map + EFF_APOS(i), 1, - EFF_SPAN_ALEN(i, afl->extras[j].len))) { + !memcmp(afl->extras[j].data, out_buf + i, afl->extras[j].len)) { --afl->stage_max; continue; @@ -1693,6 +1642,10 @@ skip_interest: for (i = 0; i <= (u32)len; ++i) { + if (!skip_eff_map[i % len]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + afl->stage_cur_byte = i; for (j = 0; j < afl->extras_cnt; ++j) { @@ -1755,6 +1708,10 @@ skip_user_extras: u32 last_len = 0; + if (!skip_eff_map[i]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + afl->stage_cur_byte = i; u32 min_extra_len = MIN(afl->a_extras_cnt, (u32)USE_AUTO_EXTRAS); @@ -1763,9 +1720,7 @@ skip_user_extras: /* See the comment in the earlier code; extras are sorted by size. */ if (afl->a_extras[j].len > len - i || - !memcmp(afl->a_extras[j].data, out_buf + i, afl->a_extras[j].len) || - !memchr(eff_map + EFF_APOS(i), 1, - EFF_SPAN_ALEN(i, afl->a_extras[j].len))) { + !memcmp(afl->a_extras[j].data, out_buf + i, afl->a_extras[j].len)) { --afl->stage_max; continue; @@ -1813,6 +1768,10 @@ skip_user_extras: for (i = 0; i <= (u32)len; ++i) { + if (!skip_eff_map[i % len]) continue; + + if (is_det_timeout(before_det_time, 0)) { goto custom_mutator_stage; } + afl->stage_cur_byte = i; for (j = 0; j < afl->a_extras_cnt; ++j) { @@ -2020,6 +1979,19 @@ custom_mutator_stage: havoc_stage: +#ifdef INTROSPECTION + + if (!is_logged) { + + is_logged = 1; + before_havoc_findings = afl->queued_items; + before_havoc_edges = count_non_255_bytes(afl, afl->virgin_bits); + before_havoc_time = get_cur_time(); + + } + +#endif + if (unlikely(afl->custom_only)) { /* Force UI update */ @@ -3430,6 +3402,25 @@ retry_splicing: ret_val = 0; +#ifdef INTROSPECTION + + afl->havoc_prof->queued_det_stage = + before_havoc_findings - before_det_findings; + afl->havoc_prof->queued_havoc_stage = + afl->queued_items - before_havoc_findings; + afl->havoc_prof->total_queued_det += afl->havoc_prof->queued_det_stage; + afl->havoc_prof->edge_det_stage = before_havoc_edges - before_det_edges; + afl->havoc_prof->edge_havoc_stage = + count_non_255_bytes(afl, afl->virgin_bits) - before_havoc_edges; + afl->havoc_prof->total_det_edge += afl->havoc_prof->edge_det_stage; + afl->havoc_prof->det_stage_time = before_havoc_time - before_det_time; + afl->havoc_prof->havoc_stage_time = get_cur_time() - before_havoc_time; + afl->havoc_prof->total_det_time += afl->havoc_prof->det_stage_time; + + plot_profile_data(afl, afl->queue_cur); + +#endif + /* we are through with this queue entry - for this iteration */ abandon_entry: diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 4c7da774..16a398fd 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 4b9627f7..1ea50418 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: @@ -664,6 +664,8 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { } + q->skipdet_e = (struct skipdet_entry *)ck_alloc(sizeof(struct skipdet_entry)); + } /* Destroy the entire queue. */ @@ -679,6 +681,15 @@ void destroy_queue(afl_state_t *afl) { q = afl->queue_buf[i]; ck_free(q->fname); ck_free(q->trace_mini); + if (q->skipdet_e) { + + if (q->skipdet_e->done_inf_map) ck_free(q->skipdet_e->done_inf_map); + if (q->skipdet_e->skip_eff_map) ck_free(q->skipdet_e->skip_eff_map); + + ck_free(q->skipdet_e); + + } + ck_free(q); } diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 9e9b3822..eead7a8b 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -11,7 +11,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 1ee8ebe7..d764952c 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -10,7 +10,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-fuzz-skipdet.c b/src/afl-fuzz-skipdet.c new file mode 100644 index 00000000..e52d59a3 --- /dev/null +++ b/src/afl-fuzz-skipdet.c @@ -0,0 +1,403 @@ + + +#include "afl-fuzz.h" + +void flip_range(u8 *input, u32 pos, u32 size) { + + for (u32 i = 0; i < size; i++) + input[pos + i] ^= 0xFF; + + return; + +} + +#define MAX_EFF_TIMEOUT (10 * 60 * 1000) +#define MAX_DET_TIMEOUT (15 * 60 * 1000) +u8 is_det_timeout(u64 cur_ms, u8 is_flip) { + + if (is_flip) { + + if (unlikely(get_cur_time() - cur_ms > MAX_EFF_TIMEOUT)) return 1; + + } else { + + if (unlikely(get_cur_time() - cur_ms > MAX_DET_TIMEOUT)) return 1; + + } + + return 0; + +} + +/* decide if the seed should be deterministically fuzzed */ + +u8 should_det_fuzz(afl_state_t *afl, struct queue_entry *q) { + + if (!afl->skipdet_g->virgin_det_bits) { + + afl->skipdet_g->virgin_det_bits = + (u8 *)ck_alloc(sizeof(u8) * afl->fsrv.map_size); + + } + + if (!q->favored || q->passed_det) return 0; + if (!q->trace_mini) return 0; + + if (!afl->skipdet_g->last_cov_undet) + afl->skipdet_g->last_cov_undet = get_cur_time(); + + if (get_cur_time() - afl->skipdet_g->last_cov_undet >= THRESHOLD_DEC_TIME) { + + if (afl->skipdet_g->undet_bits_threshold >= 2) { + + afl->skipdet_g->undet_bits_threshold *= 0.75; + afl->skipdet_g->last_cov_undet = get_cur_time(); + + } + + } + + u32 new_det_bits = 0; + + for (u32 i = 0; i < afl->fsrv.map_size; i++) { + + if (unlikely(q->trace_mini[i >> 3] & (1 << (i & 7)))) { + + if (!afl->skipdet_g->virgin_det_bits[i]) { new_det_bits++; } + + } + + } + + if (!afl->skipdet_g->undet_bits_threshold) + afl->skipdet_g->undet_bits_threshold = new_det_bits * 0.05; + + if (new_det_bits >= afl->skipdet_g->undet_bits_threshold) { + + afl->skipdet_g->last_cov_undet = get_cur_time(); + q->skipdet_e->undet_bits = new_det_bits; + + for (u32 i = 0; i < afl->fsrv.map_size; i++) { + + if (unlikely(q->trace_mini[i >> 3] & (1 << (i & 7)))) { + + if (!afl->skipdet_g->virgin_det_bits[i]) + afl->skipdet_g->virgin_det_bits[i] = 1; + + } + + } + + return 1; + + } + + return 0; + +} + +/* + consists of two stages that + return 0 if exec failed. +*/ + +u8 skip_deterministic_stage(afl_state_t *afl, u8 *orig_buf, u8 *out_buf, + u32 len, u64 before_det_time) { + + u64 orig_hit_cnt, new_hit_cnt; + + if (afl->queue_cur->skipdet_e->done_eff) return 1; + + if (!should_det_fuzz(afl, afl->queue_cur)) return 1; + + /* Add check to make sure that for seeds without too much undet bits, + we ignore them */ + + /****************** + * SKIP INFERENCE * + ******************/ + + afl->stage_short = "inf"; + afl->stage_name = "inference"; + afl->stage_cur = 0; + orig_hit_cnt = afl->queued_items + afl->saved_crashes; + + u8 *inf_eff_map = (u8 *)ck_alloc(sizeof(u8) * len); + memset(inf_eff_map, 1, sizeof(u8) * len); + + if (common_fuzz_stuff(afl, orig_buf, len)) { return 0; } + + u64 prev_cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); + u64 _prev_cksum = prev_cksum; + + if (MINIMAL_BLOCK_SIZE * 8 < len) { + + // u64 size_skiped = 0, quick_skip_exec = total_execs, quick_skip_time = + // get_cur_time(); + u64 pre_inf_exec = afl->fsrv.total_execs, pre_inf_time = get_cur_time(); + + /* if determine stage time / input size is too small, just go ahead */ + + u32 pos = 0, cur_block_size = MINIMAL_BLOCK_SIZE, max_block_size = len / 8; + + while (pos < len - 1) { + + cur_block_size = MINIMAL_BLOCK_SIZE; + + while (cur_block_size < max_block_size) { + + u32 flip_block_size = + (cur_block_size + pos < len) ? cur_block_size : len - 1 - pos; + + afl->stage_cur += 1; + + flip_range(out_buf, pos, flip_block_size); + + if (common_fuzz_stuff(afl, out_buf, len)) return 0; + + flip_range(out_buf, pos, flip_block_size); + + u64 cksum = + hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); + + // printf("Now trying range %d with %d, %s.\n", pos, cur_block_size, + // (cksum == prev_cksum) ? (u8*)"Yes" : (u8*) "Not"); + + /* continue until we fail or exceed length */ + if (cksum == _prev_cksum) { + + cur_block_size *= 2; + + if (cur_block_size >= len - 1 - pos) break; + + } else { + + break; + + } + + } + + if (cur_block_size == MINIMAL_BLOCK_SIZE) { + + /* we failed early on*/ + + pos += cur_block_size; + + } else { + + u32 cur_skip_len = (cur_block_size / 2 + pos < len) + ? (cur_block_size / 2) + : (len - pos - 1); + + memset(inf_eff_map + pos, 0, cur_skip_len); + + afl->skipdet_g->inf_prof->inf_skipped_bytes += cur_skip_len; + + pos += cur_skip_len; + + } + + } + + afl->skipdet_g->inf_prof->inf_execs_cost += + (afl->fsrv.total_execs - pre_inf_exec); + afl->skipdet_g->inf_prof->inf_time_cost += (get_cur_time() - pre_inf_time); + // PFATAL("Done, now have %d bytes skipped, with exec %lld, time %lld.\n", + // afl->inf_skipped_bytes, afl->inf_execs_cost, afl->inf_time_cost); + + } else + + memset(inf_eff_map, 1, len); + + new_hit_cnt = afl->queued_items + afl->saved_crashes; + + afl->stage_finds[STAGE_INF] += new_hit_cnt - orig_hit_cnt; + afl->stage_cycles[STAGE_INF] += afl->stage_cur; + + /**************************** + * Quick Skip Effective Map * + ****************************/ + + /* Quick Effective Map Calculation */ + + afl->stage_short = "quick"; + afl->stage_name = "quick eff"; + afl->stage_cur = 0; + afl->stage_max = 32 * 1024; + + orig_hit_cnt = afl->queued_items + afl->saved_crashes; + + u32 before_skip_inf = afl->queued_items; + + /* clean all the eff bytes, since previous eff bytes are already fuzzed */ + u8 *skip_eff_map = afl->queue_cur->skipdet_e->skip_eff_map, + *done_inf_map = afl->queue_cur->skipdet_e->done_inf_map; + + if (!skip_eff_map) { + + skip_eff_map = (u8 *)ck_alloc(sizeof(u8) * len); + afl->queue_cur->skipdet_e->skip_eff_map = skip_eff_map; + + } else { + + memset(skip_eff_map, 0, sizeof(u8) * len); + + } + + /* restore the starting point */ + if (!done_inf_map) { + + done_inf_map = (u8 *)ck_alloc(sizeof(u8) * len); + afl->queue_cur->skipdet_e->done_inf_map = done_inf_map; + + } else { + + for (afl->stage_cur = 0; afl->stage_cur < len; afl->stage_cur++) { + + if (done_inf_map[afl->stage_cur] == 0) break; + + } + + } + + /* depending on the seed's performance, we could search eff bytes + for multiple rounds */ + + u8 eff_round_continue = 1, eff_round_done = 0, done_eff = 0, repeat_eff = 0, + fuzz_nearby = 0, *non_eff_bytes = 0; + + u64 before_eff_execs = afl->fsrv.total_execs; + + if (getenv("REPEAT_EFF")) repeat_eff = 1; + if (getenv("FUZZ_NEARBY")) fuzz_nearby = 1; + + if (fuzz_nearby) { + + non_eff_bytes = (u8 *)ck_alloc(sizeof(u8) * len); + + // clean exec cksum + if (common_fuzz_stuff(afl, out_buf, len)) { return 0; } + prev_cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); + + } + + do { + + eff_round_continue = 0; + afl->stage_max = 32 * 1024; + + for (; afl->stage_cur < afl->stage_max && afl->stage_cur < len; + ++afl->stage_cur) { + + afl->stage_cur_byte = afl->stage_cur; + + if (!inf_eff_map[afl->stage_cur_byte] || + skip_eff_map[afl->stage_cur_byte]) + continue; + + if (is_det_timeout(before_det_time, 1)) { goto cleanup_skipdet; } + + u8 orig = out_buf[afl->stage_cur_byte], replace = rand_below(afl, 256); + + while (replace == orig) { + + replace = rand_below(afl, 256); + + } + + out_buf[afl->stage_cur_byte] = replace; + + before_skip_inf = afl->queued_items; + + if (common_fuzz_stuff(afl, out_buf, len)) { return 0; } + + out_buf[afl->stage_cur_byte] = orig; + + if (fuzz_nearby) { + + if (prev_cksum == + hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST)) { + + non_eff_bytes[afl->stage_cur_byte] = 1; + + } + + } + + if (afl->queued_items != before_skip_inf) { + + skip_eff_map[afl->stage_cur_byte] = 1; + afl->queue_cur->skipdet_e->quick_eff_bytes += 1; + + if (afl->stage_max < MAXIMUM_QUICK_EFF_EXECS) { afl->stage_max *= 2; } + + if (afl->stage_max == MAXIMUM_QUICK_EFF_EXECS && repeat_eff) + eff_round_continue = 1; + + } + + done_inf_map[afl->stage_cur_byte] = 1; + + } + + afl->stage_cur = 0; + done_eff = 1; + + if (++eff_round_done >= 8) break; + + } while (eff_round_continue); + + new_hit_cnt = afl->queued_items + afl->saved_crashes; + + afl->stage_finds[STAGE_QUICK] += new_hit_cnt - orig_hit_cnt; + afl->stage_cycles[STAGE_QUICK] += (afl->fsrv.total_execs - before_eff_execs); + +cleanup_skipdet: + + if (fuzz_nearby) { + + u8 *nearby_bytes = (u8 *)ck_alloc(sizeof(u8) * len); + + u32 i = 3; + while (i < len) { + + // assume DWORD size, from i - 3 -> i + 3 + if (skip_eff_map[i]) { + + u32 fill_length = (i + 3 < len) ? 7 : len - i + 2; + memset(nearby_bytes + i - 3, 1, fill_length); + i += 3; + + } else + + i += 1; + + } + + for (i = 0; i < len; i++) { + + if (nearby_bytes[i] && !non_eff_bytes[i]) skip_eff_map[i] = 1; + + } + + ck_free(nearby_bytes); + ck_free(non_eff_bytes); + + } + + if (done_eff) { + + afl->queue_cur->skipdet_e->continue_inf = 0; + afl->queue_cur->skipdet_e->done_eff = 1; + + } else { + + afl->queue_cur->skipdet_e->continue_inf = 1; + + } + + return 1; + +} + diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 7d6fdfb9..4467cae8 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -140,6 +140,14 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->fsrv.child_pid = -1; afl->fsrv.out_dir_fd = -1; + /* Init SkipDet */ + afl->skipdet_g = + (struct skipdet_global *)ck_alloc(sizeof(struct skipdet_global)); + afl->skipdet_g->inf_prof = + (struct inf_profile *)ck_alloc(sizeof(struct inf_profile)); + afl->havoc_prof = + (struct havoc_profile *)ck_alloc(sizeof(struct havoc_profile)); + init_mopt_globals(afl); list_append(&afl_states, afl); diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index deb28b7a..76577081 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -502,6 +502,44 @@ void maybe_update_plot_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, } +/* Log deterministic stage efficiency */ + +void plot_profile_data(afl_state_t *afl, struct queue_entry *q) { + + u64 current_ms = get_cur_time() - afl->start_time; + + u32 current_edges = count_non_255_bytes(afl, afl->virgin_bits); + double det_finding_rate = (double)afl->havoc_prof->total_det_edge * 100.0 / + (double)current_edges, + det_time_rate = (double)afl->havoc_prof->total_det_time * 100.0 / + (double)current_ms; + + u32 ndet_bits = 0; + for (u32 i = 0; i < afl->fsrv.map_size; i++) { + + if (afl->skipdet_g->virgin_det_bits[i]) ndet_bits += 1; + + } + + double det_fuzzed_rate = (double)ndet_bits * 100.0 / (double)current_edges; + + fprintf(afl->fsrv.det_plot_file, + "[%02lld:%02lld:%02lld] fuzz %d (%d), find %d/%d among %d(%02.2f) " + "and spend %lld/%lld(%02.2f), cover %02.2f yet, %d/%d undet bits, " + "continue %d.\n", + current_ms / 1000 / 3600, (current_ms / 1000 / 60) % 60, + (current_ms / 1000) % 60, afl->current_entry, q->fuzz_level, + afl->havoc_prof->edge_det_stage, afl->havoc_prof->edge_havoc_stage, + current_edges, det_finding_rate, + afl->havoc_prof->det_stage_time / 1000, + afl->havoc_prof->havoc_stage_time / 1000, det_time_rate, + det_fuzzed_rate, q->skipdet_e->undet_bits, + afl->skipdet_g->undet_bits_threshold, q->skipdet_e->continue_inf); + + fflush(afl->fsrv.det_plot_file); + +} + /* Check terminal dimensions after resize. */ static void check_term_size(afl_state_t *afl) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 8cf6c735..12d67fe7 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -170,7 +170,7 @@ static void usage(u8 *argv0, int more_help) { " -g minlength - set min length of generated fuzz input (default: 1)\n" " -G maxlength - set max length of generated fuzz input (default: " "%lu)\n" - " -D - enable deterministic fuzzing (once per queue entry)\n" + " -D - enable (a new) effective deterministic fuzzing\n" " -L minutes - use MOpt(imize) mode and set the time limit for " "entering the\n" " pacemaker mode (minutes of no new finds). 0 = " @@ -955,14 +955,20 @@ int main(int argc, char **argv_orig, char **envp) { break; - case 'D': /* enforce deterministic */ + case 'D': /* partial deterministic */ afl->skip_deterministic = 0; break; - case 'd': /* skip deterministic */ + case 'd': /* no deterministic */ - afl->skip_deterministic = 1; + // this is the default and currently a lot of infrastructure enforces + // it (e.g. clusterfuzz, fuzzbench) based on that this feature + // originally was bad performance wise. We now have a better + // implementation, hence if it is activated, we do not want to + // deactivate it by such setups. + + // afl->skip_deterministic = 1; break; case 'B': /* load bitmap */ @@ -1424,11 +1430,11 @@ int main(int argc, char **argv_orig, char **envp) { } #endif + + // silently disable deterministic mutation if custom mutators are used if (!afl->skip_deterministic && afl->afl_env.afl_custom_mutator_only) { - FATAL( - "Using -D determinstic fuzzing is incompatible with " - "AFL_CUSTOM_MUTATOR_ONLY!"); + afl->skip_deterministic = 1; } @@ -3031,6 +3037,11 @@ stop_fuzzing: if (frida_afl_preload) { ck_free(frida_afl_preload); } fclose(afl->fsrv.plot_file); + + #ifdef INTROSPECTION + fclose(afl->fsrv.det_plot_file); + #endif + destroy_queue(afl); destroy_extras(afl); destroy_custom_mutators(afl); diff --git a/src/afl-gotcpu.c b/src/afl-gotcpu.c index 4f851099..7aee2985 100644 --- a/src/afl-gotcpu.c +++ b/src/afl-gotcpu.c @@ -9,7 +9,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-ld-lto.c b/src/afl-ld-lto.c index 7ce5de41..513c1ae9 100644 --- a/src/afl-ld-lto.c +++ b/src/afl-ld-lto.c @@ -9,7 +9,7 @@ Andrea Fioraldi Dominik Maier - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-sharedmem.c b/src/afl-sharedmem.c index a2c81586..daea8f46 100644 --- a/src/afl-sharedmem.c +++ b/src/afl-sharedmem.c @@ -11,7 +11,7 @@ Andrea Fioraldi Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 7a639cf6..20ba5a5e 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/afl-tmin.c b/src/afl-tmin.c index e7442d1d..4e5dab41 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/test-instr.c b/test-instr.c index eda5189c..28552893 100644 --- a/test-instr.c +++ b/test-instr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/test/test-custom-mutators.sh b/test/test-custom-mutators.sh index 49feedc0..8c8b0ad3 100755 --- a/test/test-custom-mutators.sh +++ b/test/test-custom-mutators.sh @@ -38,7 +38,7 @@ test -e test-custom-mutator.c -a -e ${CUSTOM_MUTATOR_PATH}/example.c -a -e ${CUS # Run afl-fuzz w/ the C mutator $ECHO "$GREY[*] running afl-fuzz for the C mutator, this will take approx 10 seconds" { - AFL_CUSTOM_MUTATOR_LIBRARY=./libexamplemutator.so AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V07 -m ${MEM_LIMIT} -i in -o out -- ./test-custom-mutator >>errors 2>&1 + AFL_CUSTOM_MUTATOR_LIBRARY=./libexamplemutator.so AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V07 -m ${MEM_LIMIT} -i in -o out -d -- ./test-custom-mutator >>errors 2>&1 } >>errors 2>&1 # Check results @@ -58,7 +58,7 @@ test -e test-custom-mutator.c -a -e ${CUSTOM_MUTATOR_PATH}/example.c -a -e ${CUS # Run afl-fuzz w/ multiple C mutators $ECHO "$GREY[*] running afl-fuzz with multiple custom C mutators, this will take approx 10 seconds" { - AFL_CUSTOM_MUTATOR_LIBRARY="./libexamplemutator.so;./libexamplemutator2.so" AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V07 -m ${MEM_LIMIT} -i in -o out -- ./test-multiple-mutators >>errors 2>&1 + AFL_CUSTOM_MUTATOR_LIBRARY="./libexamplemutator.so;./libexamplemutator2.so" AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V07 -m ${MEM_LIMIT} -i in -o out -d -- ./test-multiple-mutators >>errors 2>&1 } >>errors 2>&1 test -n "$( ls out/default/crashes/id:000000* 2>/dev/null )" && { # TODO: update here diff --git a/unicorn_mode/build_unicorn_support.sh b/unicorn_mode/build_unicorn_support.sh index d3d16ad5..baca2171 100755 --- a/unicorn_mode/build_unicorn_support.sh +++ b/unicorn_mode/build_unicorn_support.sh @@ -14,7 +14,7 @@ # # # Copyright 2017 Battelle Memorial Institute. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/utils/afl_network_proxy/afl-network-client.c b/utils/afl_network_proxy/afl-network-client.c index 0416f0f9..1f04dd87 100644 --- a/utils/afl_network_proxy/afl-network-client.c +++ b/utils/afl_network_proxy/afl-network-client.c @@ -4,7 +4,7 @@ Written by Marc Heuse - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_network_proxy/afl-network-server.c b/utils/afl_network_proxy/afl-network-server.c index 95b0a551..c4a700f4 100644 --- a/utils/afl_network_proxy/afl-network-server.c +++ b/utils/afl_network_proxy/afl-network-server.c @@ -12,7 +12,7 @@ Dominik Maier Copyright 2016, 2017 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_proxy/afl-proxy.c b/utils/afl_proxy/afl-proxy.c index 531a97a2..6cf47636 100644 --- a/utils/afl_proxy/afl-proxy.c +++ b/utils/afl_proxy/afl-proxy.c @@ -4,7 +4,7 @@ Written by Marc Heuse - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_untracer/afl-untracer.c b/utils/afl_untracer/afl-untracer.c index 0e3f8a45..e6a74518 100644 --- a/utils/afl_untracer/afl-untracer.c +++ b/utils/afl_untracer/afl-untracer.c @@ -4,7 +4,7 @@ Written by Marc Heuse - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/afl_untracer/libtestinstr.c b/utils/afl_untracer/libtestinstr.c index b7afc325..0a98778a 100644 --- a/utils/afl_untracer/libtestinstr.c +++ b/utils/afl_untracer/libtestinstr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: diff --git a/utils/argv_fuzzing/Makefile b/utils/argv_fuzzing/Makefile index 6786467a..ba977e5f 100644 --- a/utils/argv_fuzzing/Makefile +++ b/utils/argv_fuzzing/Makefile @@ -2,7 +2,7 @@ # american fuzzy lop++ - argvfuzz # -------------------------------- # -# Copyright 2019-2023 Kjell Braden +# Copyright 2019-2024 Kjell Braden # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/utils/argv_fuzzing/argvfuzz.c b/utils/argv_fuzzing/argvfuzz.c index 41eead0c..47383138 100644 --- a/utils/argv_fuzzing/argvfuzz.c +++ b/utils/argv_fuzzing/argvfuzz.c @@ -2,7 +2,7 @@ american fuzzy lop++ - LD_PRELOAD for fuzzing argv in binaries ------------------------------------------------------------ - Copyright 2019-2023 Kjell Braden + Copyright 2019-2024 Kjell Braden Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/distributed_fuzzing/sync_script.sh b/utils/distributed_fuzzing/sync_script.sh index b22816f1..861b65c8 100755 --- a/utils/distributed_fuzzing/sync_script.sh +++ b/utils/distributed_fuzzing/sync_script.sh @@ -6,7 +6,7 @@ # Originally written by Michal Zalewski # # Copyright 2014 Google Inc. All rights reserved. -# Copyright 2019-2023 AFLplusplus Project. All rights reserved. +# Copyright 2019-2024 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/utils/libdislocator/libdislocator.so.c b/utils/libdislocator/libdislocator.so.c index 1cd7abc6..b80be1a1 100644 --- a/utils/libdislocator/libdislocator.so.c +++ b/utils/libdislocator/libdislocator.so.c @@ -6,7 +6,7 @@ Originally written by Michal Zalewski Copyright 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/libtokencap/libtokencap.so.c b/utils/libtokencap/libtokencap.so.c index f4024799..cc499150 100644 --- a/utils/libtokencap/libtokencap.so.c +++ b/utils/libtokencap/libtokencap.so.c @@ -6,7 +6,7 @@ Originally written by Michal Zalewski Copyright 2016 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/utils/persistent_mode/test-instr.c b/utils/persistent_mode/test-instr.c index 4ead6577..72e26e93 100644 --- a/utils/persistent_mode/test-instr.c +++ b/utils/persistent_mode/test-instr.c @@ -3,7 +3,7 @@ -------------------------------------------------------- Originally written by Michal Zalewski Copyright 2014 Google Inc. All rights reserved. - Copyright 2019-2023 AFLplusplus Project. All rights reserved. + Copyright 2019-2024 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: -- cgit 1.4.1 From 023fc19ce04bffcbd623e27a1f2d1810c3ec0c3c Mon Sep 17 00:00:00 2001 From: Davide Quarta Date: Mon, 5 Feb 2024 18:26:46 +0100 Subject: better replay mode error handling, added replay mode documentation, code formatting --- include/afl-fuzz.h | 2 +- include/config.h | 10 +- include/persistent_replay.h | 152 +++++++++++++++++++--------- instrumentation/README.persistent_mode.md | 30 +++++- instrumentation/afl-compiler-rt.o.c | 58 ++++++----- src/afl-forkserver.c | 5 +- utils/persistent_mode/persistent_demo_new.c | 6 +- 7 files changed, 182 insertions(+), 81 deletions(-) (limited to 'utils') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 864bc6b6..f95dcc20 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -125,7 +125,7 @@ #endif /* ^!SIMPLE_FILES */ #ifdef AFL_PERSISTENT_RECORD - #define RECORD_PREFIX "RECORD:" + #define RECORD_PREFIX "RECORD:" #endif #define STAGE_BUF_SIZE (64) /* usable size for stage name buf in afl_state */ diff --git a/include/config.h b/include/config.h index 1649f110..d44cda9c 100644 --- a/include/config.h +++ b/include/config.h @@ -83,10 +83,14 @@ will be kept and written to the crash/ directory as RECORD:... files. Note that every crash will be written, not only unique ones! */ -// #define AFL_PERSISTENT_RECORD +#define AFL_PERSISTENT_RECORD -/* Builds compiler-rt with support to replay persistent records */ -// #define AFL_PERSISTENT_REPLAY +/* Adds support in compiler-rt to replay persistent records */ +#define AFL_PERSISTENT_REPLAY + +/* Adds support in compiler-rt to replay persistent records in @@-style + * harnesses */ +// #define AFL_PERSISTENT_REPLAY_ARGPARSE /* console output colors: There are three ways to configure its behavior * 1. default: colored outputs fixed on: defined USE_COLOR && defined diff --git a/include/persistent_replay.h b/include/persistent_replay.h index b1a55e9f..58b22fb4 100644 --- a/include/persistent_replay.h +++ b/include/persistent_replay.h @@ -11,71 +11,116 @@ #include static unsigned short int is_replay_record; -static unsigned int replay_record; -static unsigned int replay_record_cnt; -static char replay_record_path[PATH_MAX]; -static char **record_arg; -static char *replay_record_dir; -static struct dirent **record_list; +static unsigned int replay_record; +static unsigned int replay_record_cnt; +static char replay_record_path[PATH_MAX]; +static char *replay_record_dir; +static struct dirent **record_list; + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE +static char **record_arg = NULL; +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE static int select_files(const struct dirent *dirbuf) { char fn[4096]; - if (dirbuf->d_name[0] == '.'){ + if (dirbuf->d_name[0] == '.') { + return 0; + } else { + snprintf(fn, sizeof(fn), "RECORD:%06u", replay_record); return !!strstr(dirbuf->d_name, fn); + } + } - + static int compare_files(const struct dirent **da, const struct dirent **db) { - - unsigned int c1=0, c2=0; + + unsigned int c1 = 0, c2 = 0; sscanf((*da)->d_name, "RECORD:%*u,cnt:%06u", &c1); sscanf((*db)->d_name, "RECORD:%*u,cnt:%06u", &c2); - return c1-c2; + return c1 - c2; + } -__attribute__((destructor)) static void __afl_record_replay_destroy(void){ - for (int i=0; i < replay_record_cnt; i++) { +__attribute__((destructor)) static void __afl_record_replay_destroy(void) { + + for (int i = 0; i < replay_record_cnt; i++) { + free(record_list[i]); + } + free(record_list); + } -__attribute__((constructor)) static void __afl_record_replay_init(int argc, char **argv) { - +__attribute__((constructor)) static void __afl_record_replay_init( +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + int argc, char **argv +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE +) { + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE char **argp; +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + + struct stat sb; + + /* caveat: if harness uses @@ and we don't pass it, it will regardless loop + * the number of iterations defined for AFL_LOOP (on the same file)*/ + if (!(is_replay_record = !!getenv("AFL_PERSISTENT_REPLAY"))) { - /* caveat: if harness uses @@ and we don't pass it, it will regardless loop the number of iterations defined for AFL_LOOP (on the same file)*/ - if(!(is_replay_record = !!getenv("AFL_PERSISTENT_REPLAY"))){ // printf("[warning] AFL_PERSISTENT_REPLAY not set.\n"); return; + } replay_record = atoi(getenv("AFL_PERSISTENT_REPLAY")); replay_record_dir = getenv("AFL_PERSISTENT_DIR"); - replay_record_cnt = scandir(replay_record_dir ? replay_record_dir : "./", &record_list, select_files, compare_files); - if (!replay_record_cnt){ - printf("[error] Can't find the requested record!\n"); + if (!(stat(replay_record_dir, &sb) == 0 && S_ISDIR(sb.st_mode))) { + + fprintf(stderr, "[error] Can't find the requested record directory!\n"); is_replay_record = 0; + return; + } + replay_record_cnt = scandir(replay_record_dir ? replay_record_dir : "./", + &record_list, select_files, compare_files); + + if (!replay_record_cnt) { + + fprintf(stderr, "[error] Can't find the requested record!\n"); + is_replay_record = 0; + + } + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE argp = argv; - while (*argp){ - if (!strcmp(*argp, "@@")){ + while (*argp) { + + if (!strcmp(*argp, "@@")) { + record_arg = argp; *record_arg = replay_record_path; break; + } + ++argp; + } +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + } /* only used if explictly included for compatibility @@ -83,67 +128,80 @@ __attribute__((constructor)) static void __afl_record_replay_init(int argc, char #ifdef AFL_COMPAT -#ifndef PATH_MAX - #define PATH_MAX 4096 -#endif + #ifndef PATH_MAX + #define PATH_MAX 4096 + #endif -#define FUZZ_BUF_SIZE 1024000 + #define FUZZ_BUF_SIZE 1024000 -// extern ssize_t read(int fildes, void *buf, size_t nbyte); + // extern ssize_t read(int fildes, void *buf, size_t nbyte); -//extern int __afl_persistent_loop(unsigned int max_cnt); -//extern unsigned char fuzz_buf[]; + // extern int __afl_persistent_loop(unsigned int max_cnt); + // extern unsigned char fuzz_buf[]; -#ifndef __AFL_HAVE_MANUAL_CONTROL - #define __AFL_HAVE_MANUAL_CONTROL -#endif + #ifndef __AFL_HAVE_MANUAL_CONTROL + #define __AFL_HAVE_MANUAL_CONTROL + #endif -#define __AFL_FUZZ_TESTCASE_LEN (read(0, fuzz_buf, FUZZ_BUF_SIZE)) -#define __AFL_FUZZ_TESTCASE_BUF fuzz_buf -#define __AFL_FUZZ_INIT() void sync(void); -#define __AFL_INIT() sync() -#define __AFL_LOOP(x) __afl_persistent_loop(x) + #define __AFL_FUZZ_TESTCASE_LEN (read(0, fuzz_buf, FUZZ_BUF_SIZE)) + #define __AFL_FUZZ_TESTCASE_BUF fuzz_buf + #define __AFL_FUZZ_INIT() void sync(void); + #define __AFL_INIT() sync() + #define __AFL_LOOP(x) __afl_persistent_loop(x) unsigned char fuzz_buf[FUZZ_BUF_SIZE]; int __afl_persistent_loop(unsigned int max_cnt) { - static unsigned int cycle_cnt = 1; + static unsigned int cycle_cnt = 1; static unsigned short int inited = 0; - char tcase[PATH_MAX]; + char tcase[PATH_MAX]; + + if (is_replay_record) { - if( is_replay_record ){ + if (!inited) { - if (!inited){ cycle_cnt = replay_record_cnt; inited = 1; + } snprintf(tcase, PATH_MAX, "%s/%s", - replay_record_dir ? replay_record_dir : "./", - record_list[replay_record_cnt-cycle_cnt]->d_name); - + replay_record_dir ? replay_record_dir : "./", + record_list[replay_record_cnt - cycle_cnt]->d_name); + #ifdef AFL_PERSISTENT_REPLAY_ARGPARSE if (record_arg) { + *record_arg = tcase; - } else { + + } else + + #endif // AFL_PERSISTENT_REPLAY_ARGPARSE + { + int fd = open(tcase, O_RDONLY); dup2(fd, 0); close(fd); + } } else { - if (!inited){ + if (!inited) { + cycle_cnt = max_cnt; inited = 1; + } } return cycle_cnt--; + } #endif // AFL_COMPAT -#endif // _HAVE_PERSISTENT_REPLAY_H \ No newline at end of file +#endif // _HAVE_PERSISTENT_REPLAY_H + diff --git a/instrumentation/README.persistent_mode.md b/instrumentation/README.persistent_mode.md index 14e59f4a..b5d982b0 100644 --- a/instrumentation/README.persistent_mode.md +++ b/instrumentation/README.persistent_mode.md @@ -195,4 +195,32 @@ Then as first line after the `__AFL_LOOP` while loop: int len = __AFL_FUZZ_TESTCASE_LEN; ``` -And that is all! \ No newline at end of file +And that is all! + +## 6) Persistent record, and replay + +If your software under test requires keeping a state between persistent loop iterations (i.e., a stateful network stack), you can use the `AFL_PERSISTENT_RECORD` variable as described in the [environment variables documentation](../docs/env_variables.md). + +To easily replay a crashing, or hanging record, you can use the persistent replay functionality by compiling AFL++ after uncommenting the `AFL_PERSISTENT_REPLAY` define in [config.h](../include/config.h). + +You can then run the test binary specifying the record number via the AFL_PERSISTENT_REPLAY environment variable (i.e., `RECORD:XXXXX`` -> `AFL_PERSISTENT_REPLAY=XXXXX`). +The directory where the record files live can be specified via the `AFL_PERSISTENT_DIR` environment varilable, otherwise by default it will be considered the current directory (`./`). + +If your harness reads the input files from arguments using the special `@@` argument you will need to define `AFL_PERSISTENT_ARGPARSE` in `config.h`, or before including the `persistent_replay.h` header file as show before. +In order to offer transparent support to harnesses using the `@@` command line argument, arguments are parsed by the `__afl_record_replay_init` init function. Since not all systems support passing arguments to initializers, this functionality is disabled by default, it's recommendable to use the `__AFL_FUZZ_TESTCASE_BUF/__AFL_FUZZ_TESTCASE_LEN` shared memory mechanism instead. + +### 7) Drop in replay functionality + +To use the replay functionality without having to use `afl-cc` you can just define `AFL_COMPAT` and include the [include/persistent_replay.h](../include/persistent_replay.h) self contained header file that provides a drop-in replacement for the persistent loop mechanism. + +```c +#ifndef __AFL_FUZZ_TESTCASE_LEN + #define AFL_COMPAT + // #define AFL_PERSISTENT_REPLAY_ARGPARSE + #include "persistent_replay.h" +#endif + +__AFL_FUZZ_INIT(); +``` + +A simple example is provided in [persistent_demo_new.c](../utils/persistent_mode/persistent_demo_new.c). \ No newline at end of file diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 0fa22aee..037caaf0 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -84,7 +84,7 @@ #include #ifdef AFL_PERSISTENT_REPLAY -#include "persistent_replay.h" + #include "persistent_replay.h" #endif /* Globals needed by the injected instrumentation. The __afl_area_initial region @@ -1344,37 +1344,49 @@ int __afl_persistent_loop(unsigned int max_cnt) { #ifdef AFL_PERSISTENT_REPLAY -#ifndef PATH_MAX - #define PATH_MAX 4096 -#endif + #ifndef PATH_MAX + #define PATH_MAX 4096 + #endif - static u8 inited = 0; - char tcase[PATH_MAX]; + static u8 inited = 0; + char tcase[PATH_MAX]; - if( unlikely(is_replay_record) ){ + if (unlikely(is_replay_record)) { - if (!inited){ - cycle_cnt = replay_record_cnt; - inited = 1; - } + if (!inited) { - snprintf(tcase, PATH_MAX, "%s/%s", - replay_record_dir ? replay_record_dir : "./", - record_list[replay_record_cnt-cycle_cnt]->d_name); + cycle_cnt = replay_record_cnt; + inited = 1; + + } + + snprintf(tcase, PATH_MAX, "%s/%s", + replay_record_dir ? replay_record_dir : "./", + record_list[replay_record_cnt - cycle_cnt]->d_name); + + #ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + if (record_arg) { + + *record_arg = tcase; + + } else + + #endif // AFL_PERSISTENT_REPLAY_ARGPARSE + { + + int fd = open(tcase, O_RDONLY); + dup2(fd, 0); + close(fd); + + } - if (record_arg) { - *record_arg = tcase; - } else { - int fd = open(tcase, O_RDONLY); - dup2(fd, 0); - close(fd); - } return cycle_cnt--; + } else -#endif +#endif - if (first_pass) { + if (first_pass) { /* Make sure that every iteration of __AFL_LOOP() starts with a clean slate. On subsequent calls, the parent will take care of that, but on the first diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index f8dd783f..36e46444 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -1593,7 +1593,7 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, #ifdef AFL_PERSISTENT_RECORD fsrv_run_result_t retval = FSRV_RUN_OK; - char *persistent_out_fmt; + char *persistent_out_fmt; #endif #ifdef __linux__ @@ -1803,6 +1803,7 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, persistent_out_fmt = "%s/hangs/RECORD:%06u,cnt:%06u"; goto store_persistent_record; #endif + } /* Did we crash? @@ -1841,7 +1842,7 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, #ifdef AFL_PERSISTENT_RECORD store_persistent_record: - if (unlikely(retval == FSRV_RUN_CRASH || retval == FSRV_RUN_TMOUT) && + if (unlikely(retval == FSRV_RUN_CRASH || retval == FSRV_RUN_TMOUT) && unlikely(fsrv->persistent_record)) { char fn[PATH_MAX]; diff --git a/utils/persistent_mode/persistent_demo_new.c b/utils/persistent_mode/persistent_demo_new.c index 40ada9e1..3d9d90a6 100644 --- a/utils/persistent_mode/persistent_demo_new.c +++ b/utils/persistent_mode/persistent_demo_new.c @@ -31,8 +31,8 @@ /* this lets the source compile without afl-clang-fast/lto */ #ifndef __AFL_FUZZ_TESTCASE_LEN -#define AFL_COMPAT -#include "persistent_replay.h" + #define AFL_COMPAT + #include "persistent_replay.h" #endif __AFL_FUZZ_INIT(); @@ -86,8 +86,6 @@ int main(int argc, char **argv) { if (buf[5] == '!') { printf("six\n"); - char *nullo = NULL+1; - *nullo = 'p'; abort(); } -- cgit 1.4.1 From 58aa181d012698471d003983aaf9e67cf92961d2 Mon Sep 17 00:00:00 2001 From: Davide Quarta Date: Tue, 6 Feb 2024 14:00:37 +0100 Subject: revert persistent_demo_new.c to b99bbf671b7469a5aad29898fe28489004c4cbe7 --- utils/persistent_mode/persistent_demo_new.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'utils') diff --git a/utils/persistent_mode/persistent_demo_new.c b/utils/persistent_mode/persistent_demo_new.c index 3d9d90a6..285f50aa 100644 --- a/utils/persistent_mode/persistent_demo_new.c +++ b/utils/persistent_mode/persistent_demo_new.c @@ -31,8 +31,17 @@ /* this lets the source compile without afl-clang-fast/lto */ #ifndef __AFL_FUZZ_TESTCASE_LEN - #define AFL_COMPAT - #include "persistent_replay.h" + +ssize_t fuzz_len; +unsigned char fuzz_buf[1024000]; + + #define __AFL_FUZZ_TESTCASE_LEN fuzz_len + #define __AFL_FUZZ_TESTCASE_BUF fuzz_buf + #define __AFL_FUZZ_INIT() void sync(void); + #define __AFL_LOOP(x) \ + ((fuzz_len = read(0, fuzz_buf, sizeof(fuzz_buf))) > 0 ? 1 : 0) + #define __AFL_INIT() sync() + #endif __AFL_FUZZ_INIT(); -- cgit 1.4.1 From e405e721fad46e594b633147a6940cfdd602e4c2 Mon Sep 17 00:00:00 2001 From: Davide Quarta Date: Tue, 6 Feb 2024 18:19:52 +0100 Subject: reuse first_pass aux var in persistent loop for record replay mode, keep area ptr and loc logic intact in record replay mode, move replay record example to own dir in utils, update docs, move record compat layer to separate header file --- include/afl-persistent-replay.h | 131 +++++++++++++++++ include/afl-record-compat.h | 67 +++++++++ include/config.h | 6 +- include/persistent_replay.h | 207 --------------------------- instrumentation/README.persistent_mode.md | 20 +-- instrumentation/afl-compiler-rt.o.c | 87 +++++------ utils/persistent_mode/Makefile | 1 - utils/replay_record/Makefile | 8 ++ utils/replay_record/persistent_demo_replay.c | 148 +++++++++++++++++++ 9 files changed, 413 insertions(+), 262 deletions(-) create mode 100644 include/afl-persistent-replay.h create mode 100644 include/afl-record-compat.h delete mode 100644 include/persistent_replay.h create mode 100644 utils/replay_record/Makefile create mode 100644 utils/replay_record/persistent_demo_replay.c (limited to 'utils') diff --git a/include/afl-persistent-replay.h b/include/afl-persistent-replay.h new file mode 100644 index 00000000..9e60ff9c --- /dev/null +++ b/include/afl-persistent-replay.h @@ -0,0 +1,131 @@ +#ifndef _HAVE_PERSISTENT_REPLAY_H +#define _HAVE_PERSISTENT_REPLAY_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PATH_MAX + #define PATH_MAX 4096 +#endif + +static unsigned short int is_replay_record; +static unsigned int replay_record; +static unsigned int replay_record_cnt; +static char replay_record_path[PATH_MAX]; +static char *replay_record_dir; +static struct dirent **record_list; + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE +static char **record_arg = NULL; +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + +static int select_files(const struct dirent *dirbuf) { + + char fn[PATH_MAX]; + + if (dirbuf->d_name[0] == '.') { + + return 0; + + } else { + + snprintf(fn, sizeof(fn), "RECORD:%06u", replay_record); + return !!strstr(dirbuf->d_name, fn); + + } + +} + +static int compare_files(const struct dirent **da, const struct dirent **db) { + + unsigned int c1 = 0, c2 = 0; + + sscanf((*da)->d_name, "RECORD:%*u,cnt:%06u", &c1); + sscanf((*db)->d_name, "RECORD:%*u,cnt:%06u", &c2); + + return c1 - c2; + +} + +__attribute__((destructor)) static void __afl_record_replay_destroy(void) { + + for (int i = 0; i < replay_record_cnt; i++) { + + free(record_list[i]); + + } + + free(record_list); + +} + +__attribute__((constructor)) static void __afl_record_replay_init( +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + int argc, char **argv +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE +) { + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + char **argp; +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + + struct stat sb; + + /* caveat: if harness uses @@ and we don't pass it, it will regardless loop + * the number of iterations defined for AFL_LOOP (on the same file)*/ + if (!(is_replay_record = !!getenv("AFL_PERSISTENT_REPLAY"))) { + + // printf("[warning] AFL_PERSISTENT_REPLAY not set.\n"); + return; + + } + + replay_record = atoi(getenv("AFL_PERSISTENT_REPLAY")); + replay_record_dir = getenv("AFL_PERSISTENT_DIR"); + + if (!(stat(replay_record_dir, &sb) == 0 && S_ISDIR(sb.st_mode))) { + + fprintf(stderr, "[error] Can't find the requested record directory!\n"); + is_replay_record = 0; + return; + + } + + replay_record_cnt = scandir(replay_record_dir ? replay_record_dir : "./", + &record_list, select_files, compare_files); + + if (!replay_record_cnt) { + + fprintf(stderr, "[error] Can't find the requested record!\n"); + is_replay_record = 0; + + } + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + argp = argv; + while (*argp) { + + if (!strcmp(*argp, "@@")) { + + record_arg = argp; + *record_arg = replay_record_path; + break; + + } + + ++argp; + + } + +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + +} + +#endif // _HAVE_PERSISTENT_REPLAY_H + diff --git a/include/afl-record-compat.h b/include/afl-record-compat.h new file mode 100644 index 00000000..2c79595d --- /dev/null +++ b/include/afl-record-compat.h @@ -0,0 +1,67 @@ +#ifndef _HAVE_AFL_COMPAT_H +#define _HAVE_AFL_COMPAT_H + +#include + +#define FUZZ_BUF_SIZE 1024000 + +// extern ssize_t read(int fildes, void *buf, size_t nbyte); + +// extern int __afl_persistent_loop(unsigned int max_cnt); +// extern unsigned char fuzz_buf[]; + +#ifndef __AFL_HAVE_MANUAL_CONTROL + #define __AFL_HAVE_MANUAL_CONTROL +#endif + +#define __AFL_FUZZ_TESTCASE_LEN (read(0, fuzz_buf, FUZZ_BUF_SIZE)) +#define __AFL_FUZZ_TESTCASE_BUF fuzz_buf +#define __AFL_FUZZ_INIT() void sync(void); +#define __AFL_INIT() sync() +#define __AFL_LOOP(x) __afl_persistent_loop(x) + +unsigned char fuzz_buf[FUZZ_BUF_SIZE]; + +int __afl_persistent_loop(unsigned int max_cnt) { + + static unsigned int cycle_cnt = 1; + static unsigned short int inited = 0; + char tcase[PATH_MAX]; + + if (is_replay_record) { + + if (!inited) { + + cycle_cnt = replay_record_cnt; + inited = 1; + + } + + snprintf(tcase, PATH_MAX, "%s/%s", + replay_record_dir ? replay_record_dir : "./", + record_list[replay_record_cnt - cycle_cnt]->d_name); + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + if (record_arg) { + + *record_arg = tcase; + + } else + +#endif // AFL_PERSISTENT_REPLAY_ARGPARSE + { + + int fd = open(tcase, O_RDONLY); + dup2(fd, 0); + close(fd); + + } + + } + + return --cycle_cnt; + +} + +#endif // _HAVE_AFL_COMPAT_H + diff --git a/include/config.h b/include/config.h index d44cda9c..a5b6eba1 100644 --- a/include/config.h +++ b/include/config.h @@ -83,13 +83,11 @@ will be kept and written to the crash/ directory as RECORD:... files. Note that every crash will be written, not only unique ones! */ -#define AFL_PERSISTENT_RECORD - -/* Adds support in compiler-rt to replay persistent records */ -#define AFL_PERSISTENT_REPLAY +// #define AFL_PERSISTENT_RECORD /* Adds support in compiler-rt to replay persistent records in @@-style * harnesses */ + // #define AFL_PERSISTENT_REPLAY_ARGPARSE /* console output colors: There are three ways to configure its behavior diff --git a/include/persistent_replay.h b/include/persistent_replay.h deleted file mode 100644 index 58b22fb4..00000000 --- a/include/persistent_replay.h +++ /dev/null @@ -1,207 +0,0 @@ -#ifndef _HAVE_PERSISTENT_REPLAY_H -#define _HAVE_PERSISTENT_REPLAY_H - -#include -#include -#include -#include -#include -#include -#include -#include - -static unsigned short int is_replay_record; -static unsigned int replay_record; -static unsigned int replay_record_cnt; -static char replay_record_path[PATH_MAX]; -static char *replay_record_dir; -static struct dirent **record_list; - -#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE -static char **record_arg = NULL; -#endif // AFL_PERSISTENT_REPLAY_ARGPARSE - -static int select_files(const struct dirent *dirbuf) { - - char fn[4096]; - - if (dirbuf->d_name[0] == '.') { - - return 0; - - } else { - - snprintf(fn, sizeof(fn), "RECORD:%06u", replay_record); - return !!strstr(dirbuf->d_name, fn); - - } - -} - -static int compare_files(const struct dirent **da, const struct dirent **db) { - - unsigned int c1 = 0, c2 = 0; - - sscanf((*da)->d_name, "RECORD:%*u,cnt:%06u", &c1); - sscanf((*db)->d_name, "RECORD:%*u,cnt:%06u", &c2); - - return c1 - c2; - -} - -__attribute__((destructor)) static void __afl_record_replay_destroy(void) { - - for (int i = 0; i < replay_record_cnt; i++) { - - free(record_list[i]); - - } - - free(record_list); - -} - -__attribute__((constructor)) static void __afl_record_replay_init( -#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE - int argc, char **argv -#endif // AFL_PERSISTENT_REPLAY_ARGPARSE -) { - -#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE - char **argp; -#endif // AFL_PERSISTENT_REPLAY_ARGPARSE - - struct stat sb; - - /* caveat: if harness uses @@ and we don't pass it, it will regardless loop - * the number of iterations defined for AFL_LOOP (on the same file)*/ - if (!(is_replay_record = !!getenv("AFL_PERSISTENT_REPLAY"))) { - - // printf("[warning] AFL_PERSISTENT_REPLAY not set.\n"); - return; - - } - - replay_record = atoi(getenv("AFL_PERSISTENT_REPLAY")); - replay_record_dir = getenv("AFL_PERSISTENT_DIR"); - - if (!(stat(replay_record_dir, &sb) == 0 && S_ISDIR(sb.st_mode))) { - - fprintf(stderr, "[error] Can't find the requested record directory!\n"); - is_replay_record = 0; - return; - - } - - replay_record_cnt = scandir(replay_record_dir ? replay_record_dir : "./", - &record_list, select_files, compare_files); - - if (!replay_record_cnt) { - - fprintf(stderr, "[error] Can't find the requested record!\n"); - is_replay_record = 0; - - } - -#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE - argp = argv; - while (*argp) { - - if (!strcmp(*argp, "@@")) { - - record_arg = argp; - *record_arg = replay_record_path; - break; - - } - - ++argp; - - } - -#endif // AFL_PERSISTENT_REPLAY_ARGPARSE - -} - -/* only used if explictly included for compatibility - compiling without afl-cc */ - -#ifdef AFL_COMPAT - - #ifndef PATH_MAX - #define PATH_MAX 4096 - #endif - - #define FUZZ_BUF_SIZE 1024000 - - // extern ssize_t read(int fildes, void *buf, size_t nbyte); - - // extern int __afl_persistent_loop(unsigned int max_cnt); - // extern unsigned char fuzz_buf[]; - - #ifndef __AFL_HAVE_MANUAL_CONTROL - #define __AFL_HAVE_MANUAL_CONTROL - #endif - - #define __AFL_FUZZ_TESTCASE_LEN (read(0, fuzz_buf, FUZZ_BUF_SIZE)) - #define __AFL_FUZZ_TESTCASE_BUF fuzz_buf - #define __AFL_FUZZ_INIT() void sync(void); - #define __AFL_INIT() sync() - #define __AFL_LOOP(x) __afl_persistent_loop(x) - -unsigned char fuzz_buf[FUZZ_BUF_SIZE]; - -int __afl_persistent_loop(unsigned int max_cnt) { - - static unsigned int cycle_cnt = 1; - static unsigned short int inited = 0; - char tcase[PATH_MAX]; - - if (is_replay_record) { - - if (!inited) { - - cycle_cnt = replay_record_cnt; - inited = 1; - - } - - snprintf(tcase, PATH_MAX, "%s/%s", - replay_record_dir ? replay_record_dir : "./", - record_list[replay_record_cnt - cycle_cnt]->d_name); - - #ifdef AFL_PERSISTENT_REPLAY_ARGPARSE - if (record_arg) { - - *record_arg = tcase; - - } else - - #endif // AFL_PERSISTENT_REPLAY_ARGPARSE - { - - int fd = open(tcase, O_RDONLY); - dup2(fd, 0); - close(fd); - - } - - } else { - - if (!inited) { - - cycle_cnt = max_cnt; - inited = 1; - - } - - } - - return cycle_cnt--; - -} - -#endif // AFL_COMPAT - -#endif // _HAVE_PERSISTENT_REPLAY_H - diff --git a/instrumentation/README.persistent_mode.md b/instrumentation/README.persistent_mode.md index b5d982b0..8e4f6ae4 100644 --- a/instrumentation/README.persistent_mode.md +++ b/instrumentation/README.persistent_mode.md @@ -201,26 +201,28 @@ And that is all! If your software under test requires keeping a state between persistent loop iterations (i.e., a stateful network stack), you can use the `AFL_PERSISTENT_RECORD` variable as described in the [environment variables documentation](../docs/env_variables.md). -To easily replay a crashing, or hanging record, you can use the persistent replay functionality by compiling AFL++ after uncommenting the `AFL_PERSISTENT_REPLAY` define in [config.h](../include/config.h). - -You can then run the test binary specifying the record number via the AFL_PERSISTENT_REPLAY environment variable (i.e., `RECORD:XXXXX`` -> `AFL_PERSISTENT_REPLAY=XXXXX`). +When `AFL_PERSISTENT_RECORD` is enabled, replay functionality is also included in the compiler-rt library. To replay a specific record, assign the record number to the AFL_PERSISTENT_REPLAY environment variable (i.e., `RECORD:XXXXX`` -> `AFL_PERSISTENT_REPLAY=XXXXX`), and run the test binary as you would normally do. The directory where the record files live can be specified via the `AFL_PERSISTENT_DIR` environment varilable, otherwise by default it will be considered the current directory (`./`). -If your harness reads the input files from arguments using the special `@@` argument you will need to define `AFL_PERSISTENT_ARGPARSE` in `config.h`, or before including the `persistent_replay.h` header file as show before. +If your harness reads the input files from arguments using the special `@@` argument you will need to include support by enabling `AFL_PERSISTENT_ARGPARSE` in `config.h`. + In order to offer transparent support to harnesses using the `@@` command line argument, arguments are parsed by the `__afl_record_replay_init` init function. Since not all systems support passing arguments to initializers, this functionality is disabled by default, it's recommendable to use the `__AFL_FUZZ_TESTCASE_BUF/__AFL_FUZZ_TESTCASE_LEN` shared memory mechanism instead. -### 7) Drop in replay functionality +## 7) Drop-in persistent loop replay replacement -To use the replay functionality without having to use `afl-cc` you can just define `AFL_COMPAT` and include the [include/persistent_replay.h](../include/persistent_replay.h) self contained header file that provides a drop-in replacement for the persistent loop mechanism. +To use the replay functionality without having to use `afl-cc`, include the [include/record_compat.h](../include/afl-record_compat.h) header file. Together with the [include/afl-persistent-replay.h](../include/afl-persistent-replay.h) header included in it, `afl-record-compat.h` provides a drop-in replacement for the persistent loop mechanism. ```c #ifndef __AFL_FUZZ_TESTCASE_LEN - #define AFL_COMPAT // #define AFL_PERSISTENT_REPLAY_ARGPARSE - #include "persistent_replay.h" + #include "afl-record-compat.h" #endif __AFL_FUZZ_INIT(); ``` -A simple example is provided in [persistent_demo_new.c](../utils/persistent_mode/persistent_demo_new.c). \ No newline at end of file +A simple example is provided in [persistent_demo_replay.c](../utils/replay_record/persistent_demo_replay.c). + +Be aware that the [afl-record-compat.h](../include/afl-record-compat.h) header should only be included in a single compilation unit, or you will end up with clobbered functions and variables. + +If you need a cleaner solution, you'll have to move the functions and variables defined in [include/record_compat.h](../include/afl-record-compat.h) and [include/afl-persistent-replay.h](../include/afl-persistent-replay.h) in a C file, and add the relevant declarations to a header file. After including the new header file, the compilation unit resulting from compiling the C file can then be linked with your program. \ No newline at end of file diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 037caaf0..4c5d4e79 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -83,8 +83,8 @@ #include #include -#ifdef AFL_PERSISTENT_REPLAY - #include "persistent_replay.h" +#ifdef AFL_PERSISTENT_RECORD + #include "afl-persistent-replay.h" #endif /* Globals needed by the injected instrumentation. The __afl_area_initial region @@ -1342,68 +1342,73 @@ int __afl_persistent_loop(unsigned int max_cnt) { static u8 first_pass = 1; static u32 cycle_cnt; -#ifdef AFL_PERSISTENT_REPLAY +#ifdef AFL_PERSISTENT_RECORD + char tcase[PATH_MAX]; +#endif - #ifndef PATH_MAX - #define PATH_MAX 4096 - #endif + if (first_pass) { - static u8 inited = 0; - char tcase[PATH_MAX]; + /* Make sure that every iteration of __AFL_LOOP() starts with a clean slate. + On subsequent calls, the parent will take care of that, but on the first + iteration, it's our job to erase any trace of whatever happened + before the loop. */ - if (unlikely(is_replay_record)) { + memset(__afl_area_ptr, 0, __afl_map_size); + __afl_area_ptr[0] = 1; + memset(__afl_prev_loc, 0, NGRAM_SIZE_MAX * sizeof(PREV_LOC_T)); - if (!inited) { + first_pass = 0; + __afl_selective_coverage_temp = 1; + +#ifdef AFL_PERSISTENT_RECORD + if (unlikely(is_replay_record)) { cycle_cnt = replay_record_cnt; - inited = 1; + goto persistent_record; - } + } else - snprintf(tcase, PATH_MAX, "%s/%s", - replay_record_dir ? replay_record_dir : "./", - record_list[replay_record_cnt - cycle_cnt]->d_name); +#endif + { - #ifdef AFL_PERSISTENT_REPLAY_ARGPARSE - if (record_arg) { + cycle_cnt = max_cnt; - *record_arg = tcase; + } - } else + return 1; - #endif // AFL_PERSISTENT_REPLAY_ARGPARSE - { + } else if (--cycle_cnt) { - int fd = open(tcase, O_RDONLY); - dup2(fd, 0); - close(fd); +#ifdef AFL_PERSISTENT_RECORD + if (unlikely(is_replay_record)) { - } + persistent_record: - return cycle_cnt--; + snprintf(tcase, PATH_MAX, "%s/%s", + replay_record_dir ? replay_record_dir : "./", + record_list[replay_record_cnt - cycle_cnt]->d_name); - } else + #ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + if (unlikely(record_arg)) { -#endif + *record_arg = tcase; - if (first_pass) { + } else - /* Make sure that every iteration of __AFL_LOOP() starts with a clean slate. - On subsequent calls, the parent will take care of that, but on the first - iteration, it's our job to erase any trace of whatever happened - before the loop. */ + #endif // AFL_PERSISTENT_REPLAY_ARGPARSE + { - memset(__afl_area_ptr, 0, __afl_map_size); - __afl_area_ptr[0] = 1; - memset(__afl_prev_loc, 0, NGRAM_SIZE_MAX * sizeof(PREV_LOC_T)); + int fd = open(tcase, O_RDONLY); + dup2(fd, 0); + close(fd); - cycle_cnt = max_cnt; - first_pass = 0; - __afl_selective_coverage_temp = 1; + } - return 1; + return 1; - } else if (--cycle_cnt) { + } + +#endif raise(SIGSTOP); diff --git a/utils/persistent_mode/Makefile b/utils/persistent_mode/Makefile index 64de82a7..498aa3f8 100644 --- a/utils/persistent_mode/Makefile +++ b/utils/persistent_mode/Makefile @@ -1,7 +1,6 @@ all: ../../afl-clang-fast -o persistent_demo persistent_demo.c ../../afl-clang-fast -o persistent_demo_new persistent_demo_new.c - gcc -g -I ../../include -o persistent_demo_new_compat persistent_demo_new.c AFL_DONT_OPTIMIZE=1 ../../afl-clang-fast -o test-instr test-instr.c document: diff --git a/utils/replay_record/Makefile b/utils/replay_record/Makefile new file mode 100644 index 00000000..0d1cba92 --- /dev/null +++ b/utils/replay_record/Makefile @@ -0,0 +1,8 @@ +all: + test `grep '//[\s\t ]*#define[\s\t ]*AFL_PERSISTENT_RECORD' ../../include/config.h | wc -l` -eq 0 || (echo "AFL_PERSISTENT_RECORD must be enabled in config.h"; exit 1) + ../../afl-clang-fast -o persistent_demo_replay persistent_demo_replay.c + ${CC} -I ../../include -o persistent_demo_replay_compat persistent_demo_replay.c + ${CC} -g -I ../../include -DAFL_PERSISTENT_REPLAY_ARGPARSE -o persistent_demo_replay_argparse persistent_demo_replay.c + +clean: + rm -f persistent_demo_replay persistent_demo_replay_argparse persistent_demo_replay_compat diff --git a/utils/replay_record/persistent_demo_replay.c b/utils/replay_record/persistent_demo_replay.c new file mode 100644 index 00000000..6f6648f1 --- /dev/null +++ b/utils/replay_record/persistent_demo_replay.c @@ -0,0 +1,148 @@ +/* + american fuzzy lop++ - persistent mode example + -------------------------------------------- + + Originally written by Michal Zalewski + + Copyright 2015 Google Inc. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + + This file demonstrates the high-performance "persistent mode" that may be + suitable for fuzzing certain fast and well-behaved libraries, provided that + they are stateless or that their internal state can be easily reset + across runs. + + To make this work, the library and this shim need to be compiled in LLVM + mode using afl-clang-fast (other compiler wrappers will *not* work). + + */ + +#include +#include +#include +#include +#include +#include + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + #include + #include +#endif + +/* this lets the source compile without afl-clang-fast/lto */ +#ifndef __AFL_FUZZ_TESTCASE_LEN + #include "afl-record-compat.h" +#endif + +__AFL_FUZZ_INIT(); + +/* Main entry point. */ + +/* To ensure checks are not optimized out it is recommended to disable + code optimization for the fuzzer harness main() */ +#pragma clang optimize off +#pragma GCC optimize("O0") + +int main(int argc, char **argv) { + + ssize_t len; /* how much input did we read? */ + unsigned char *buf; /* test case buffer pointer */ + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + int fd; + + if (argc < 2) { printf("Need an input file!"); } +#endif + + /* The number passed to __AFL_LOOP() controls the maximum number of + iterations before the loop exits and the program is allowed to + terminate normally. This limits the impact of accidental memory leaks + and similar hiccups. */ + + __AFL_INIT(); + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + buf = malloc(1000); +#else + buf = __AFL_FUZZ_TESTCASE_BUF; // this must be assigned before __AFL_LOOP! +#endif + + while (__AFL_LOOP(UINT_MAX)) { // increase if you have good stability + +#ifdef AFL_PERSISTENT_REPLAY_ARGPARSE + fd = open(argv[1], O_RDONLY); + len = read(fd, buf, 1000); + close(fd); +#else + len = __AFL_FUZZ_TESTCASE_LEN; // do not use the macro directly in a call! +#endif + + // fprintf(stderr, "input: %zd \"%s\"\n", len, buf); + + /* do we have enough data? */ + if (len < 8) continue; + + if (strcmp((char *)buf, "thisisateststring") == 0) printf("teststring\n"); + + if (buf[0] == 'f') { + + printf("one\n"); + if (buf[1] == 'o') { + + printf("two\n"); + if (buf[2] == 'o') { + + printf("three\n"); + if (buf[3] == '!') { + + printf("four\n"); + if (buf[4] == '!') { + + printf("five\n"); + if (buf[5] == '!') { + + printf("six\n"); + abort(); + + } else { + + if (buf[5] == 'O') { + + // hang + while (1) { + + continue; + + }; + + } + + } + + } + + } + + } + + } + + } + + /*** END PLACEHOLDER CODE ***/ + + } + + /* Once the loop is exited, terminate normally - AFL will restart the process + when this happens, with a clean slate when it comes to allocated memory, + leftover file descriptors, etc. */ + + return 0; + +} + -- cgit 1.4.1 From 8e4bd0314e0ee2cd4b758e96591b0338a534623a Mon Sep 17 00:00:00 2001 From: Davide Quarta Date: Tue, 6 Feb 2024 21:34:56 +0100 Subject: added README.md to utils/replay_record --- utils/replay_record/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 utils/replay_record/README.md (limited to 'utils') diff --git a/utils/replay_record/README.md b/utils/replay_record/README.md new file mode 100644 index 00000000..f015850d --- /dev/null +++ b/utils/replay_record/README.md @@ -0,0 +1,9 @@ +# AFL++ persistent record replay + +Persistent demo replay showcases the `AFL_PERSISTENT_RECORD` replay functionality. +The Makefile will produce three binaries: + + persistent_demo_replay: uses afl-cc and makes use of the replay functionality included in the compiler runtime library + + persistent_demo_replay_compat: uses the [afl-record-compat.h](../../include/afl-record-compat.h) compatibility header to compile the same example without `afl-cc` + + persistent_demo_replay_argparse: makes use of `afl-record-compat.h`, and the Makefile defines `AFL_PERSISTENT_REPLAY_ARGPARSE` to test the replay functionality but parses the input file via a command-line argument (`@@`-style harness). + +For more information see [README.persistent_mode.md](../../instrumentation/README.persistent_mode.md). \ No newline at end of file -- cgit 1.4.1 From 956fa95d77ac3cbc43cd44b56bffc605e2a2090e Mon Sep 17 00:00:00 2001 From: Davide Quarta Date: Wed, 7 Feb 2024 12:00:11 +0100 Subject: updated readme --- utils/replay_record/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'utils') diff --git a/utils/replay_record/README.md b/utils/replay_record/README.md index f015850d..6d72ca97 100644 --- a/utils/replay_record/README.md +++ b/utils/replay_record/README.md @@ -1,7 +1,8 @@ # AFL++ persistent record replay -Persistent demo replay showcases the `AFL_PERSISTENT_RECORD` replay functionality. -The Makefile will produce three binaries: +This persistent record replay demo showcases the `AFL_PERSISTENT_RECORD` replay functionality. + +The [Makefile](Makefile) will produce three binaries: + persistent_demo_replay: uses afl-cc and makes use of the replay functionality included in the compiler runtime library + persistent_demo_replay_compat: uses the [afl-record-compat.h](../../include/afl-record-compat.h) compatibility header to compile the same example without `afl-cc` + persistent_demo_replay_argparse: makes use of `afl-record-compat.h`, and the Makefile defines `AFL_PERSISTENT_REPLAY_ARGPARSE` to test the replay functionality but parses the input file via a command-line argument (`@@`-style harness). -- cgit 1.4.1 From 038fef962c3d85fe7e37fcd8717270654f927881 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 8 Feb 2024 12:46:00 +0100 Subject: performance --- .gitignore | 1 + GNUmakefile | 59 +- docs/INSTALL.md | 5 +- include/config.h | 2 +- include/t1ha.h | 719 +++ include/t1ha0_ia32aes_b.h | 167 + include/t1ha_bits.h | 1254 +++++ include/t1ha_selfcheck.h | 76 + include/xxhash.h | 11013 ++++++++++++++++++++++++-------------------- src/afl-fuzz.c | 4 + src/afl-performance.c | 9 + utils/bench/Makefile | 8 + utils/bench/README.md | 2 + utils/bench/hash.c | 42 + 14 files changed, 8215 insertions(+), 5146 deletions(-) create mode 100644 include/t1ha.h create mode 100644 include/t1ha0_ia32aes_b.h create mode 100644 include/t1ha_bits.h create mode 100644 include/t1ha_selfcheck.h create mode 100644 utils/bench/Makefile create mode 100644 utils/bench/README.md create mode 100644 utils/bench/hash.c (limited to 'utils') diff --git a/.gitignore b/.gitignore index 67feb240..8e191e29 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ unicorn_mode/samples/*/\.test-* utils/afl_network_proxy/afl-network-client utils/afl_network_proxy/afl-network-server utils/afl_proxy/afl-proxy +utils/bench/hash utils/optimin/build utils/optimin/optimin utils/persistent_mode/persistent_demo diff --git a/GNUmakefile b/GNUmakefile index 283c57c2..64dfc37f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -84,21 +84,27 @@ else endif endif -#ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -fno-move-loop-invariants -fdisable-tree-cunrolli -x c - -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" -# SPECIAL_PERFORMANCE += -fno-move-loop-invariants -fdisable-tree-cunrolli -#endif - -#ifeq "$(shell echo 'int main() {return 0; }' | $(CC) $(CFLAGS) -Werror -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" -# ifndef SOURCE_DATE_EPOCH -# HAVE_MARCHNATIVE = 1 -# CFLAGS_OPT += -march=native -# endif -#endif +ifdef PERFORMANCE + SPECIAL_PERFORMANCE := -D_AFL_SPECIAL_PERFORMANCE + ifeq "$(SYS)" "Linux" + ifeq "$(shell grep avx2 /proc/cpuinfo)" "" + else + SPECIAL_PERFORMANCE += -mavx2 -D_HAVE_AVX2 + endif + endif + ifeq "$(shell echo 'int main() {return 0; }' | $(CC) $(CFLAGS) -Werror -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + HAVE_MARCHNATIVE = 1 + SPECIAL_PERFORMANCE += -march=native + endif + $(info SPECIAL_PERFORMANCE=$(SPECIAL_PERFORMANCE)) +else + SPECIAL_PERFORMANCE := +endif ifneq "$(SYS)" "Darwin" - #ifeq "$(HAVE_MARCHNATIVE)" "1" - # SPECIAL_PERFORMANCE += -march=native - #endif + #ifeq "$(HAVE_MARCHNATIVE)" "1" + # SPECIAL_PERFORMANCE += -march=native + #endif #ifndef DEBUG # CFLAGS_OPT += -D_FORTIFY_SOURCE=1 #endif @@ -389,6 +395,7 @@ help: @echo @echo Known build environment options: @echo "==========================================" + @echo "PERFORMANCE - compile with performance options that make the binary not transferable to other systems. Recommended!" @echo STATIC - compile AFL++ static @echo "CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md)" @echo ASAN_BUILD - compiles AFL++ with memory sanitizer for debug purposes @@ -453,31 +460,31 @@ afl-as: src/afl-as.c include/afl-as.h $(COMM_HDR) | test_x86 @ln -sf afl-as as src/afl-performance.o : $(COMM_HDR) src/afl-performance.c include/hash.h - $(CC) $(CFLAGS) $(CFLAGS_OPT) -Iinclude -c src/afl-performance.c -o src/afl-performance.o + $(CC) $(CFLAGS) $(CFLAGS_OPT) $(SPECIAL_PERFORMANCE) -Iinclude -c src/afl-performance.c -o src/afl-performance.o src/afl-common.o : $(COMM_HDR) src/afl-common.c include/common.h - $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-common.c -o src/afl-common.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) -c src/afl-common.c -o src/afl-common.o src/afl-forkserver.o : $(COMM_HDR) src/afl-forkserver.c include/forkserver.h - $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-forkserver.c -o src/afl-forkserver.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) -c src/afl-forkserver.c -o src/afl-forkserver.o src/afl-sharedmem.o : $(COMM_HDR) src/afl-sharedmem.c include/sharedmem.h - $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-sharedmem.c -o src/afl-sharedmem.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) -c src/afl-sharedmem.c -o src/afl-sharedmem.o afl-fuzz: $(COMM_HDR) include/afl-fuzz.h $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(PYFLAGS) $(LDFLAGS) -lm + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(PYFLAGS) $(LDFLAGS) -lm afl-showmap: src/afl-showmap.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-fuzz-mutators.c src/afl-fuzz-python.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(PYFLAGS) $(LDFLAGS) + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) src/$@.c src/afl-fuzz-mutators.c src/afl-fuzz-python.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(PYFLAGS) $(LDFLAGS) afl-tmin: src/afl-tmin.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(LDFLAGS) afl-analyze: src/afl-analyze.c src/afl-common.o src/afl-sharedmem.o src/afl-performance.o src/afl-forkserver.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-performance.o src/afl-forkserver.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-performance.o src/afl-forkserver.o -o $@ $(LDFLAGS) afl-gotcpu: src/afl-gotcpu.c src/afl-common.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-common.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(SPECIAL_PERFORMANCE) src/$@.c src/afl-common.o -o $@ $(LDFLAGS) .PHONY: document document: afl-fuzz-document @@ -494,17 +501,17 @@ unit_maybe_alloc: test/unittests/unit_maybe_alloc.o ./test/unittests/unit_maybe_alloc test/unittests/unit_hash.o : $(COMM_HDR) include/alloc-inl.h test/unittests/unit_hash.c $(AFL_FUZZ_FILES) src/afl-performance.o - @$(CC) $(CFLAGS) $(ASAN_CFLAGS) -c test/unittests/unit_hash.c -o test/unittests/unit_hash.o + @$(CC) $(CFLAGS) $(ASAN_CFLAGS) $(SPECIAL_PERFORMANCE) -c test/unittests/unit_hash.c -o test/unittests/unit_hash.o unit_hash: test/unittests/unit_hash.o src/afl-performance.o - @$(CC) $(CFLAGS) -Wl,--wrap=exit -Wl,--wrap=printf $^ -o test/unittests/unit_hash $(LDFLAGS) $(ASAN_LDFLAGS) -lcmocka + @$(CC) $(CFLAGS) $(SPECIAL_PERFORMANCE) -Wl,--wrap=exit -Wl,--wrap=printf $^ -o test/unittests/unit_hash $(LDFLAGS) $(ASAN_LDFLAGS) -lcmocka ./test/unittests/unit_hash test/unittests/unit_rand.o : $(COMM_HDR) include/alloc-inl.h test/unittests/unit_rand.c $(AFL_FUZZ_FILES) src/afl-performance.o - @$(CC) $(CFLAGS) $(ASAN_CFLAGS) -c test/unittests/unit_rand.c -o test/unittests/unit_rand.o + @$(CC) $(CFLAGS) $(ASAN_CFLAGS) $(SPECIAL_PERFORMANCE) -c test/unittests/unit_rand.c -o test/unittests/unit_rand.o unit_rand: test/unittests/unit_rand.o src/afl-common.o src/afl-performance.o - @$(CC) $(CFLAGS) $(ASAN_CFLAGS) -Wl,--wrap=exit -Wl,--wrap=printf $^ -o test/unittests/unit_rand $(LDFLAGS) $(ASAN_LDFLAGS) -lcmocka + @$(CC) $(CFLAGS) $(ASAN_CFLAGS) $(SPECIAL_PERFORMANCE) -Wl,--wrap=exit -Wl,--wrap=printf $^ -o test/unittests/unit_rand $(LDFLAGS) $(ASAN_LDFLAGS) -lcmocka ./test/unittests/unit_rand test/unittests/unit_list.o : $(COMM_HDR) include/list.h test/unittests/unit_list.c $(AFL_FUZZ_FILES) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 84bbe3ea..9f53afed 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -69,14 +69,15 @@ These build targets exist: [Unless you are on Mac OS X](https://developer.apple.com/library/archive/qa/qa1118/_index.html), you can also build statically linked versions of the AFL++ binaries by passing -the `STATIC=1` argument to make: +the `PERFORMANCE=1` argument to make: ```shell -make STATIC=1 +make PERFORMANCE=1 ``` These build options exist: +* PERFORMANCE - compile with performance options that make the binary not transferable to other systems. Recommended! * STATIC - compile AFL++ static * CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md) * ASAN_BUILD - compiles AFL++ with memory sanitizer for debug purposes diff --git a/include/config.h b/include/config.h index 70ce2ae3..31d66b14 100644 --- a/include/config.h +++ b/include/config.h @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.10c" +#define VERSION "++4.20a" /****************************************************** * * diff --git a/include/t1ha.h b/include/t1ha.h new file mode 100644 index 00000000..498f0dd6 --- /dev/null +++ b/include/t1ha.h @@ -0,0 +1,719 @@ +/* + * Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com, + * Fast Positive Hash. + * + * Portions Copyright (c) 2010-2020 Leonid Yuriev , + * The 1Hippeus project (t1h). + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgement in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" } + * by [Positive Technologies](https://www.ptsecurity.ru) + * + * Briefly, it is a 64-bit Hash Function: + * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64, + * but portable and without penalties it can run on any 64-bit CPU. + * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash + * and all others portable hash-functions (which do not use specific + * hardware tricks). + * 3. Not suitable for cryptography. + * + * The Future will (be) Positive. Всё будет хорошо. + * + * ACKNOWLEDGEMENT: + * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев) + * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta! + */ + +#pragma once + +/***************************************************************************** + * + * PLEASE PAY ATTENTION TO THE FOLLOWING NOTES + * about macros definitions which controls t1ha behaviour and/or performance. + * + * + * 1) T1HA_SYS_UNALIGNED_ACCESS = Defines the system/platform/CPU/architecture + * abilities for unaligned data access. + * + * By default, when the T1HA_SYS_UNALIGNED_ACCESS not defined, + * it will defined on the basis hardcoded knowledge about of capabilities + * of most common CPU architectures. But you could override this + * default behavior when build t1ha library itself: + * + * // To disable unaligned access at all. + * #define T1HA_SYS_UNALIGNED_ACCESS 0 + * + * // To enable unaligned access, but indicate that it significantly slow. + * #define T1HA_SYS_UNALIGNED_ACCESS 1 + * + * // To enable unaligned access, and indicate that it effecient. + * #define T1HA_SYS_UNALIGNED_ACCESS 2 + * + * + * 2) T1HA_USE_FAST_ONESHOT_READ = Controls the data reads at the end of buffer. + * + * When defined to non-zero, t1ha will use 'one shot' method for reading + * up to 8 bytes at the end of data. In this case just the one 64-bit read + * will be performed even when the available less than 8 bytes. + * + * This is little bit faster that switching by length of data tail. + * Unfortunately this will triggering a false-positive alarms from Valgrind, + * AddressSanitizer and other similar tool. + * + * By default, t1ha defines it to 1, but you could override this + * default behavior when build t1ha library itself: + * + * // For little bit faster and small code. + * #define T1HA_USE_FAST_ONESHOT_READ 1 + * + * // For calmness if doubt. + * #define T1HA_USE_FAST_ONESHOT_READ 0 + * + * + * 3) T1HA0_RUNTIME_SELECT = Controls choice fastest function in runtime. + * + * t1ha library offers the t1ha0() function as the fastest for current CPU. + * But actual CPU's features/capabilities and may be significantly different, + * especially on x86 platform. Therefore, internally, t1ha0() may require + * dynamic dispatching for choice best implementation. + * + * By default, t1ha enables such runtime choice and (may be) corresponding + * indirect calls if it reasonable, but you could override this default + * behavior when build t1ha library itself: + * + * // To enable runtime choice of fastest implementation. + * #define T1HA0_RUNTIME_SELECT 1 + * + * // To disable runtime choice of fastest implementation. + * #define T1HA0_RUNTIME_SELECT 0 + * + * When T1HA0_RUNTIME_SELECT is nonzero the t1ha0_resolve() function could + * be used to get actual t1ha0() implementation address at runtime. This is + * useful for two cases: + * - calling by local pointer-to-function usually is little + * bit faster (less overhead) than via a PLT thru the DSO boundary. + * - GNU Indirect functions (see below) don't supported by environment + * and calling by t1ha0_funcptr is not available and/or expensive. + * + * 4) T1HA_USE_INDIRECT_FUNCTIONS = Controls usage of GNU Indirect functions. + * + * In continue of T1HA0_RUNTIME_SELECT the T1HA_USE_INDIRECT_FUNCTIONS + * controls usage of ELF indirect functions feature. In general, when + * available, this reduces overhead of indirect function's calls though + * a DSO-bundary (https://sourceware.org/glibc/wiki/GNU_IFUNC). + * + * By default, t1ha engage GNU Indirect functions when it available + * and useful, but you could override this default behavior when build + * t1ha library itself: + * + * // To enable use of GNU ELF Indirect functions. + * #define T1HA_USE_INDIRECT_FUNCTIONS 1 + * + * // To disable use of GNU ELF Indirect functions. This may be useful + * // if the actual toolchain or the system's loader don't support ones. + * #define T1HA_USE_INDIRECT_FUNCTIONS 0 + * + * 5) T1HA0_AESNI_AVAILABLE = Controls AES-NI detection and dispatching on x86. + * + * In continue of T1HA0_RUNTIME_SELECT the T1HA0_AESNI_AVAILABLE controls + * detection and usage of AES-NI CPU's feature. On the other hand, this + * requires compiling parts of t1ha library with certain properly options, + * and could be difficult or inconvenient in some cases. + * + * By default, t1ha engade AES-NI for t1ha0() on the x86 platform, but + * you could override this default behavior when build t1ha library itself: + * + * // To disable detection and usage of AES-NI instructions for t1ha0(). + * // This may be useful when you unable to build t1ha library properly + * // or known that AES-NI will be unavailable at the deploy. + * #define T1HA0_AESNI_AVAILABLE 0 + * + * // To force detection and usage of AES-NI instructions for t1ha0(), + * // but I don't known reasons to anybody would need this. + * #define T1HA0_AESNI_AVAILABLE 1 + * + * 6) T1HA0_DISABLED, T1HA1_DISABLED, T1HA2_DISABLED = Controls availability of + * t1ha functions. + * + * In some cases could be useful to import/use only few of t1ha functions + * or just the one. So, this definitions allows disable corresponding parts + * of t1ha library. + * + * // To disable t1ha0(), t1ha0_32le(), t1ha0_32be() and all AES-NI. + * #define T1HA0_DISABLED + * + * // To disable t1ha1_le() and t1ha1_be(). + * #define T1HA1_DISABLED + * + * // To disable t1ha2_atonce(), t1ha2_atonce128() and so on. + * #define T1HA2_DISABLED + * + *****************************************************************************/ + +#define T1HA_VERSION_MAJOR 2 +#define T1HA_VERSION_MINOR 1 +#define T1HA_VERSION_RELEASE 1 + +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __has_include +#define __has_include(x) (0) +#endif + +#ifndef __GNUC_PREREQ +#if defined(__GNUC__) && defined(__GNUC_MINOR__) +#define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#else +#define __GNUC_PREREQ(maj, min) 0 +#endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ +#ifdef __clang__ +#define __CLANG_PREREQ(maj, min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#else +#define __CLANG_PREREQ(maj, min) (0) +#endif +#endif /* __CLANG_PREREQ */ + +#ifndef __LCC_PREREQ +#ifdef __LCC__ +#define __LCC_PREREQ(maj, min) \ + ((__LCC__ << 16) + __LCC_MINOR__ >= ((maj) << 16) + (min)) +#else +#define __LCC_PREREQ(maj, min) (0) +#endif +#endif /* __LCC_PREREQ */ + +/*****************************************************************************/ + +#ifdef _MSC_VER +/* Avoid '16' bytes padding added after data member 't1ha_context::total' + * and other warnings from std-headers if warning-level > 3. */ +#pragma warning(push, 3) +#endif + +#if defined(__cplusplus) && __cplusplus >= 201103L +#include +#include +#include +#else +#include +#include +#include +#endif + +/*****************************************************************************/ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#ifndef __ia32__ +/* LY: define neutral __ia32__ for x86 and x86-64 archs */ +#define __ia32__ 1 +#endif /* __ia32__ */ +#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) +/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ +#define __amd64__ 1 +#endif /* __amd64__ */ +#endif /* all x86 */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ + +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \ + defined(HAVE_ENDIAN_H) || __has_include() +#include +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#include +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() +#include +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include() && __has_include()) +#include +#include +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NETBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() +#include +#endif /* OS */ + +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN +#define __BYTE_ORDER__ _BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 + +#if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ + defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ + defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ + defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ + defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ + defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#else +#error __BYTE_ORDER__ should be defined. +#endif /* Arch */ + +#endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/*****************************************************************************/ + +#ifndef __dll_export +#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) +#if defined(__GNUC__) || __has_attribute(dllexport) +#define __dll_export __attribute__((dllexport)) +#else +#define __dll_export __declspec(dllexport) +#endif +#elif defined(__GNUC__) || __has_attribute(__visibility__) +#define __dll_export __attribute__((__visibility__("default"))) +#else +#define __dll_export +#endif +#endif /* __dll_export */ + +#ifndef __dll_import +#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) +#if defined(__GNUC__) || __has_attribute(dllimport) +#define __dll_import __attribute__((dllimport)) +#else +#define __dll_import __declspec(dllimport) +#endif +#elif defined(__GNUC__) || __has_attribute(__visibility__) +#define __dll_import __attribute__((__visibility__("default"))) +#else +#define __dll_import +#endif +#endif /* __dll_import */ + +#ifndef __force_inline +#ifdef _MSC_VER +#define __force_inline __forceinline +#elif __GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__) +#define __force_inline __inline __attribute__((__always_inline__)) +#else +#define __force_inline __inline +#endif +#endif /* __force_inline */ + +#ifndef T1HA_API +#if defined(t1ha_EXPORTS) +#define T1HA_API __dll_export +#elif defined(t1ha_IMPORTS) +#define T1HA_API __dll_import +#else +#define T1HA_API +#endif +#endif /* T1HA_API */ + +#if defined(_MSC_VER) && defined(__ia32__) +#define T1HA_ALIGN_PREFIX __declspec(align(32)) /* required only for SIMD */ +#else +#define T1HA_ALIGN_PREFIX +#endif /* _MSC_VER */ + +#if defined(__GNUC__) && defined(__ia32__) +#define T1HA_ALIGN_SUFFIX \ + __attribute__((__aligned__(32))) /* required only for SIMD */ +#else +#define T1HA_ALIGN_SUFFIX +#endif /* GCC x86 */ + +#ifndef T1HA_USE_INDIRECT_FUNCTIONS +/* GNU ELF indirect functions usage control. For more info please see + * https://en.wikipedia.org/wiki/Executable_and_Linkable_Format + * and https://sourceware.org/glibc/wiki/GNU_IFUNC */ +#if defined(__ELF__) && defined(__amd64__) && \ + (__has_attribute(__ifunc__) || \ + (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && \ + !defined(__SANITIZE_ADDRESS__) && !defined(__SSP_ALL__))) +/* Enable gnu_indirect_function by default if : + * - ELF AND x86_64 + * - attribute(__ifunc__) is available OR + * GCC >= 4 WITHOUT -fsanitize=address NOR -fstack-protector-all */ +#define T1HA_USE_INDIRECT_FUNCTIONS 1 +#else +#define T1HA_USE_INDIRECT_FUNCTIONS 0 +#endif +#endif /* T1HA_USE_INDIRECT_FUNCTIONS */ + +#if __GNUC_PREREQ(4, 0) +#pragma GCC visibility push(hidden) +#endif /* __GNUC_PREREQ(4,0) */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef union T1HA_ALIGN_PREFIX t1ha_state256 { + uint8_t bytes[32]; + uint32_t u32[8]; + uint64_t u64[4]; + struct { + uint64_t a, b, c, d; + } n; +} t1ha_state256_t T1HA_ALIGN_SUFFIX; + +typedef struct t1ha_context { + t1ha_state256_t state; + t1ha_state256_t buffer; + size_t partial; + uint64_t total; +} t1ha_context_t; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +/****************************************************************************** + * + * Self-testing API. + * + * Unfortunately, some compilers (exactly only Microsoft Visual C/C++) has + * a bugs which leads t1ha-functions to produce wrong results. This API allows + * check the correctness of the actual code in runtime. + * + * All check-functions returns 0 on success, or -1 in case the corresponding + * hash-function failed verification. PLEASE, always perform such checking at + * initialization of your code, if you using MSVC or other troubleful compilers. + */ + +T1HA_API int t1ha_selfcheck__all_enabled(void); + +#ifndef T1HA2_DISABLED +T1HA_API int t1ha_selfcheck__t1ha2_atonce(void); +T1HA_API int t1ha_selfcheck__t1ha2_atonce128(void); +T1HA_API int t1ha_selfcheck__t1ha2_stream(void); +T1HA_API int t1ha_selfcheck__t1ha2(void); +#endif /* T1HA2_DISABLED */ + +#ifndef T1HA1_DISABLED +T1HA_API int t1ha_selfcheck__t1ha1_le(void); +T1HA_API int t1ha_selfcheck__t1ha1_be(void); +T1HA_API int t1ha_selfcheck__t1ha1(void); +#endif /* T1HA1_DISABLED */ + +#ifndef T1HA0_DISABLED +T1HA_API int t1ha_selfcheck__t1ha0_32le(void); +T1HA_API int t1ha_selfcheck__t1ha0_32be(void); +T1HA_API int t1ha_selfcheck__t1ha0(void); + +/* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */ +#ifndef T1HA0_AESNI_AVAILABLE +#if defined(__e2k__) || \ + (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800)) +#define T1HA0_AESNI_AVAILABLE 1 +#else +#define T1HA0_AESNI_AVAILABLE 0 +#endif +#endif /* ifndef T1HA0_AESNI_AVAILABLE */ + +#if T1HA0_AESNI_AVAILABLE +T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_noavx(void); +T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx(void); +#ifndef __e2k__ +T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx2(void); +#endif +#endif /* if T1HA0_AESNI_AVAILABLE */ +#endif /* T1HA0_DISABLED */ + +/****************************************************************************** + * + * t1ha2 = 64 and 128-bit, SLIGHTLY MORE ATTENTION FOR QUALITY AND STRENGTH. + * + * - The recommended version of "Fast Positive Hash" with good quality + * for checksum, hash tables and fingerprinting. + * - Portable and extremely efficiency on modern 64-bit CPUs. + * Designed for 64-bit little-endian platforms, + * in other cases will runs slowly. + * - Great quality of hashing and still faster than other non-t1ha hashes. + * Provides streaming mode and 128-bit result. + * + * Note: Due performance reason 64- and 128-bit results are completely + * different each other, i.e. 64-bit result is NOT any part of 128-bit. + */ +#ifndef T1HA2_DISABLED + +/* The at-once variant with 64-bit result */ +T1HA_API uint64_t t1ha2_atonce(const void *data, size_t length, uint64_t seed); + +/* The at-once variant with 128-bit result. + * Argument `extra_result` is NOT optional and MUST be valid. + * The high 64-bit part of 128-bit hash will be always unconditionally + * stored to the address given by `extra_result` argument. */ +T1HA_API uint64_t t1ha2_atonce128(uint64_t *__restrict extra_result, + const void *__restrict data, size_t length, + uint64_t seed); + +/* The init/update/final trinity for streaming. + * Return 64 or 128-bit result depentently from `extra_result` argument. */ +T1HA_API void t1ha2_init(t1ha_context_t *ctx, uint64_t seed_x, uint64_t seed_y); +T1HA_API void t1ha2_update(t1ha_context_t *__restrict ctx, + const void *__restrict data, size_t length); + +/* Argument `extra_result` is optional and MAY be NULL. + * - If `extra_result` is NOT NULL then the 128-bit hash will be calculated, + * and high 64-bit part of it will be stored to the address given + * by `extra_result` argument. + * - Otherwise the 64-bit hash will be calculated + * and returned from function directly. + * + * Note: Due performance reason 64- and 128-bit results are completely + * different each other, i.e. 64-bit result is NOT any part of 128-bit. */ +T1HA_API uint64_t t1ha2_final(t1ha_context_t *__restrict ctx, + uint64_t *__restrict extra_result /* optional */); + +#endif /* T1HA2_DISABLED */ + +/****************************************************************************** + * + * t1ha1 = 64-bit, BASELINE FAST PORTABLE HASH: + * + * - Runs faster on 64-bit platforms in other cases may runs slowly. + * - Portable and stable, returns same 64-bit result + * on all architectures and CPUs. + * - Unfortunately it fails the "strict avalanche criteria", + * see test results at https://github.com/demerphq/smhasher. + * + * This flaw is insignificant for the t1ha1() purposes and imperceptible + * from a practical point of view. + * However, nowadays this issue has resolved in the next t1ha2(), + * that was initially planned to providing a bit more quality. + */ +#ifndef T1HA1_DISABLED + +/* The little-endian variant. */ +T1HA_API uint64_t t1ha1_le(const void *data, size_t length, uint64_t seed); + +/* The big-endian variant. */ +T1HA_API uint64_t t1ha1_be(const void *data, size_t length, uint64_t seed); + +#endif /* T1HA1_DISABLED */ + +/****************************************************************************** + * + * t1ha0 = 64-bit, JUST ONLY FASTER: + * + * - Provides fast-as-possible hashing for current CPU, including + * 32-bit systems and engaging the available hardware acceleration. + * - It is a facade that selects most quick-and-dirty hash + * for the current processor. For instance, on IA32 (x86) actual function + * will be selected in runtime, depending on current CPU capabilities + * + * BE CAREFUL!!! THIS IS MEANS: + * + * 1. The quality of hash is a subject for tradeoffs with performance. + * So, the quality and strength of t1ha0() may be lower than t1ha1(), + * especially on 32-bit targets, but then much faster. + * However, guaranteed that it passes all SMHasher tests. + * + * 2. No warranty that the hash result will be same for particular + * key on another machine or another version of libt1ha. + * + * Briefly, such hash-results and their derivatives, should be + * used only in runtime, but should not be persist or transferred + * over a network. + * + * + * When T1HA0_RUNTIME_SELECT is nonzero the t1ha0_resolve() function could + * be used to get actual t1ha0() implementation address at runtime. This is + * useful for two cases: + * - calling by local pointer-to-function usually is little + * bit faster (less overhead) than via a PLT thru the DSO boundary. + * - GNU Indirect functions (see below) don't supported by environment + * and calling by t1ha0_funcptr is not available and/or expensive. + */ + +#ifndef T1HA0_DISABLED + +/* The little-endian variant for 32-bit CPU. */ +uint64_t t1ha0_32le(const void *data, size_t length, uint64_t seed); +/* The big-endian variant for 32-bit CPU. */ +uint64_t t1ha0_32be(const void *data, size_t length, uint64_t seed); + +/* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */ +#ifndef T1HA0_AESNI_AVAILABLE +#if defined(__e2k__) || \ + (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800)) +#define T1HA0_AESNI_AVAILABLE 1 +#else +#define T1HA0_AESNI_AVAILABLE 0 +#endif +#endif /* T1HA0_AESNI_AVAILABLE */ + +/* Define T1HA0_RUNTIME_SELECT to 0 for disable dispatching t1ha0 at runtime. */ +#ifndef T1HA0_RUNTIME_SELECT +#if T1HA0_AESNI_AVAILABLE && !defined(__e2k__) +#define T1HA0_RUNTIME_SELECT 1 +#else +#define T1HA0_RUNTIME_SELECT 0 +#endif +#endif /* T1HA0_RUNTIME_SELECT */ + +#if !T1HA0_RUNTIME_SELECT && !defined(T1HA0_USE_DEFINE) +#if defined(__LCC__) +#define T1HA0_USE_DEFINE 1 +#else +#define T1HA0_USE_DEFINE 0 +#endif +#endif /* T1HA0_USE_DEFINE */ + +#if T1HA0_AESNI_AVAILABLE +uint64_t t1ha0_ia32aes_noavx(const void *data, size_t length, uint64_t seed); +uint64_t t1ha0_ia32aes_avx(const void *data, size_t length, uint64_t seed); +#ifndef __e2k__ +uint64_t t1ha0_ia32aes_avx2(const void *data, size_t length, uint64_t seed); +#endif +#endif /* T1HA0_AESNI_AVAILABLE */ + +#if T1HA0_RUNTIME_SELECT +typedef uint64_t (*t1ha0_function_t)(const void *, size_t, uint64_t); +T1HA_API t1ha0_function_t t1ha0_resolve(void); +#if T1HA_USE_INDIRECT_FUNCTIONS +T1HA_API uint64_t t1ha0(const void *data, size_t length, uint64_t seed); +#else +/* Otherwise function pointer will be used. + * Unfortunately this may cause some overhead calling. */ +T1HA_API extern uint64_t (*t1ha0_funcptr)(const void *data, size_t length, + uint64_t seed); +static __force_inline uint64_t t1ha0(const void *data, size_t length, + uint64_t seed) { + return t1ha0_funcptr(data, length, seed); +} +#endif /* T1HA_USE_INDIRECT_FUNCTIONS */ + +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + +#if T1HA0_USE_DEFINE + +#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) +#if defined(T1HA1_DISABLED) +#define t1ha0 t1ha2_atonce +#else +#define t1ha0 t1ha1_be +#endif /* T1HA1_DISABLED */ +#else /* 32/64 */ +#define t1ha0 t1ha0_32be +#endif /* 32/64 */ + +#else /* T1HA0_USE_DEFINE */ + +static __force_inline uint64_t t1ha0(const void *data, size_t length, + uint64_t seed) { +#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) +#if defined(T1HA1_DISABLED) + return t1ha2_atonce(data, length, seed); +#else + return t1ha1_be(data, length, seed); +#endif /* T1HA1_DISABLED */ +#else /* 32/64 */ + return t1ha0_32be(data, length, seed); +#endif /* 32/64 */ +} + +#endif /* !T1HA0_USE_DEFINE */ + +#else /* !T1HA0_RUNTIME_SELECT && __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ + +#if T1HA0_USE_DEFINE + +#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) +#if defined(T1HA1_DISABLED) +#define t1ha0 t1ha2_atonce +#else +#define t1ha0 t1ha1_le +#endif /* T1HA1_DISABLED */ +#else /* 32/64 */ +#define t1ha0 t1ha0_32le +#endif /* 32/64 */ + +#else + +static __force_inline uint64_t t1ha0(const void *data, size_t length, + uint64_t seed) { +#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) +#if defined(T1HA1_DISABLED) + return t1ha2_atonce(data, length, seed); +#else + return t1ha1_le(data, length, seed); +#endif /* T1HA1_DISABLED */ +#else /* 32/64 */ + return t1ha0_32le(data, length, seed); +#endif /* 32/64 */ +} + +#endif /* !T1HA0_USE_DEFINE */ + +#endif /* !T1HA0_RUNTIME_SELECT */ + +#endif /* T1HA0_DISABLED */ + +#ifdef __cplusplus +} +#endif + +#if __GNUC_PREREQ(4, 0) +#pragma GCC visibility pop +#endif /* __GNUC_PREREQ(4,0) */ diff --git a/include/t1ha0_ia32aes_b.h b/include/t1ha0_ia32aes_b.h new file mode 100644 index 00000000..e8e52638 --- /dev/null +++ b/include/t1ha0_ia32aes_b.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com, + * Fast Positive Hash. + * + * Portions Copyright (c) 2010-2020 Leonid Yuriev , + * The 1Hippeus project (t1h). + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgement in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" } + * by [Positive Technologies](https://www.ptsecurity.ru) + * + * Briefly, it is a 64-bit Hash Function: + * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64, + * but portable and without penalties it can run on any 64-bit CPU. + * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash + * and all others portable hash-functions (which do not use specific + * hardware tricks). + * 3. Not suitable for cryptography. + * + * The Future will (be) Positive. Всё будет хорошо. + * + * ACKNOWLEDGEMENT: + * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев) + * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta! + */ + +#include "t1ha_bits.h" +#include "t1ha_selfcheck.h" + +#if T1HA0_AESNI_AVAILABLE + +uint64_t T1HA_IA32AES_NAME(const void *data, uint32_t len) { + uint64_t a = 0; + uint64_t b = len; + + if (likely(len > 32)) { + __m128i x = _mm_set_epi64x(a, b); + __m128i y = _mm_aesenc_si128(x, _mm_set_epi64x(prime_0, prime_1)); + + const __m128i *v = (const __m128i *)data; + const __m128i *const detent = + (const __m128i *)((const uint8_t *)data + (len & ~15ul)); + data = detent; + + if (len & 16) { + x = _mm_add_epi64(x, _mm_loadu_si128(v++)); + y = _mm_aesenc_si128(x, y); + } + len &= 15; + + if (v + 7 < detent) { + __m128i salt = y; + do { + __m128i t = _mm_aesenc_si128(_mm_loadu_si128(v++), salt); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); + + salt = _mm_add_epi64(salt, _mm_set_epi64x(prime_5, prime_6)); + t = _mm_aesenc_si128(x, t); + x = _mm_add_epi64(y, x); + y = t; + } while (v + 7 < detent); + } + + while (v < detent) { + __m128i v0y = _mm_add_epi64(y, _mm_loadu_si128(v++)); + __m128i v1x = _mm_sub_epi64(x, _mm_loadu_si128(v++)); + x = _mm_aesdec_si128(x, v0y); + y = _mm_aesdec_si128(y, v1x); + } + + x = _mm_add_epi64(_mm_aesdec_si128(x, _mm_aesenc_si128(y, x)), y); +#if defined(__x86_64__) || defined(_M_X64) +#if defined(__SSE4_1__) || defined(__AVX__) + a = _mm_extract_epi64(x, 0); + b = _mm_extract_epi64(x, 1); +#else + a = _mm_cvtsi128_si64(x); + b = _mm_cvtsi128_si64(_mm_unpackhi_epi64(x, x)); +#endif +#else +#if defined(__SSE4_1__) || defined(__AVX__) + a = (uint32_t)_mm_extract_epi32(x, 0) | (uint64_t)_mm_extract_epi32(x, 1) + << 32; + b = (uint32_t)_mm_extract_epi32(x, 2) | (uint64_t)_mm_extract_epi32(x, 3) + << 32; +#else + a = (uint32_t)_mm_cvtsi128_si32(x); + a |= (uint64_t)_mm_cvtsi128_si32(_mm_shuffle_epi32(x, 1)) << 32; + x = _mm_unpackhi_epi64(x, x); + b = (uint32_t)_mm_cvtsi128_si32(x); + b |= (uint64_t)_mm_cvtsi128_si32(_mm_shuffle_epi32(x, 1)) << 32; +#endif +#endif +#ifdef __AVX__ + _mm256_zeroupper(); +#elif !(defined(_X86_64_) || defined(__x86_64__) || defined(_M_X64) || \ + defined(__e2k__)) + _mm_empty(); +#endif + } + + const uint64_t *v = (const uint64_t *)data; + switch (len) { + default: + mixup64(&a, &b, fetch64_le_unaligned(v++), prime_4); + /* fall through */ + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + mixup64(&b, &a, fetch64_le_unaligned(v++), prime_3); + /* fall through */ + case 16: + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + mixup64(&a, &b, fetch64_le_unaligned(v++), prime_2); + /* fall through */ + case 8: + case 7: + case 6: + case 5: + case 4: + case 3: + case 2: + case 1: + mixup64(&b, &a, tail64_le_unaligned(v, len), prime_1); + /* fall through */ + case 0: + return final64(a, b); + } +} + +#endif /* T1HA0_AESNI_AVAILABLE */ +#undef T1HA_IA32AES_NAME diff --git a/include/t1ha_bits.h b/include/t1ha_bits.h new file mode 100644 index 00000000..539369aa --- /dev/null +++ b/include/t1ha_bits.h @@ -0,0 +1,1254 @@ +/* + * Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com, + * Fast Positive Hash. + * + * Portions Copyright (c) 2010-2020 Leonid Yuriev , + * The 1Hippeus project (t1h). + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgement in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" } + * by [Positive Technologies](https://www.ptsecurity.ru) + * + * Briefly, it is a 64-bit Hash Function: + * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64, + * but portable and without penalties it can run on any 64-bit CPU. + * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash + * and all others portable hash-functions (which do not use specific + * hardware tricks). + * 3. Not suitable for cryptography. + * + * The Future will (be) Positive. Всё будет хорошо. + * + * ACKNOWLEDGEMENT: + * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев) + * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta! + */ + +#pragma once + +#if defined(_MSC_VER) +#pragma warning(disable : 4201) /* nameless struct/union */ +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif /* 1800 */ +#endif /* MSVC */ +#include "t1ha.h" + +#ifndef T1HA_USE_FAST_ONESHOT_READ +/* Define it to 1 for little bit faster code. + * Unfortunately this may triggering a false-positive alarms from Valgrind, + * AddressSanitizer and other similar tool. + * So, define it to 0 for calmness if doubt. */ +#define T1HA_USE_FAST_ONESHOT_READ 1 +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +/*****************************************************************************/ + +#include /* for assert() */ +#include /* for bool */ +#include /* for memcpy() */ + +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ + __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ +#error Unsupported byte order. +#endif + +#define T1HA_UNALIGNED_ACCESS__UNABLE 0 +#define T1HA_UNALIGNED_ACCESS__SLOW 1 +#define T1HA_UNALIGNED_ACCESS__EFFICIENT 2 + +#ifndef T1HA_SYS_UNALIGNED_ACCESS +#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) +#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT +#elif defined(__ia32__) +#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT +#elif defined(__e2k__) +#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__SLOW +#elif defined(__ARM_FEATURE_UNALIGNED) +#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT +#else +#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__UNABLE +#endif +#endif /* T1HA_SYS_UNALIGNED_ACCESS */ + +#define ALIGNMENT_16 2 +#define ALIGNMENT_32 4 +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul +#define ALIGNMENT_64 8 +#else +#define ALIGNMENT_64 4 +#endif + +#ifndef PAGESIZE +#define PAGESIZE 4096 +#endif /* PAGESIZE */ + +/***************************************************************************/ + +#ifndef __has_builtin +#define __has_builtin(x) (0) +#endif + +#ifndef __has_warning +#define __has_warning(x) (0) +#endif + +#ifndef __has_feature +#define __has_feature(x) (0) +#endif + +#ifndef __has_extension +#define __has_extension(x) (0) +#endif + +#if __has_feature(address_sanitizer) +#define __SANITIZE_ADDRESS__ 1 +#endif + +#ifndef __optimize +#if defined(__clang__) && !__has_attribute(__optimize__) +#define __optimize(ops) +#elif defined(__GNUC__) || __has_attribute(__optimize__) +#define __optimize(ops) __attribute__((__optimize__(ops))) +#else +#define __optimize(ops) +#endif +#endif /* __optimize */ + +#ifndef __cold +#if defined(__OPTIMIZE__) +#if defined(__e2k__) +#define __cold __optimize(1) __attribute__((__cold__)) +#elif defined(__clang__) && !__has_attribute(__cold__) && \ + __has_attribute(__section__) +/* just put infrequently used functions in separate section */ +#define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") +#elif defined(__GNUC__) || __has_attribute(__cold__) +#define __cold __attribute__((__cold__)) __optimize("Os") +#else +#define __cold __optimize("Os") +#endif +#else +#define __cold +#endif +#endif /* __cold */ + +#if __GNUC_PREREQ(4, 4) || defined(__clang__) + +#if defined(__ia32__) || defined(__e2k__) +#include +#endif + +#if defined(__ia32__) && !defined(__cpuid_count) +#include +#endif + +#if defined(__e2k__) +#include +#endif + +#ifndef likely +#define likely(cond) __builtin_expect(!!(cond), 1) +#endif + +#ifndef unlikely +#define unlikely(cond) __builtin_expect(!!(cond), 0) +#endif + +#if __GNUC_PREREQ(4, 5) || __has_builtin(__builtin_unreachable) +#define unreachable() __builtin_unreachable() +#endif + +#define bswap64(v) __builtin_bswap64(v) +#define bswap32(v) __builtin_bswap32(v) +#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16) +#define bswap16(v) __builtin_bswap16(v) +#endif + +#if !defined(__maybe_unused) && \ + (__GNUC_PREREQ(4, 3) || __has_attribute(__unused__)) +#define __maybe_unused __attribute__((__unused__)) +#endif + +#if !defined(__always_inline) && \ + (__GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__)) +#define __always_inline __inline __attribute__((__always_inline__)) +#endif + +#if defined(__e2k__) + +#if __iset__ >= 3 +#define mul_64x64_high(a, b) __builtin_e2k_umulhd(a, b) +#endif /* __iset__ >= 3 */ + +#if __iset__ >= 5 +static __maybe_unused __always_inline unsigned +e2k_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) { + *sum = base + addend; + return (unsigned)__builtin_e2k_addcd_c(base, addend, 0); +} +#define add64carry_first(base, addend, sum) \ + e2k_add64carry_first(base, addend, sum) + +static __maybe_unused __always_inline unsigned +e2k_add64carry_next(unsigned carry, uint64_t base, uint64_t addend, + uint64_t *sum) { + *sum = __builtin_e2k_addcd(base, addend, carry); + return (unsigned)__builtin_e2k_addcd_c(base, addend, carry); +} +#define add64carry_next(carry, base, addend, sum) \ + e2k_add64carry_next(carry, base, addend, sum) + +static __maybe_unused __always_inline void e2k_add64carry_last(unsigned carry, + uint64_t base, + uint64_t addend, + uint64_t *sum) { + *sum = __builtin_e2k_addcd(base, addend, carry); +} +#define add64carry_last(carry, base, addend, sum) \ + e2k_add64carry_last(carry, base, addend, sum) +#endif /* __iset__ >= 5 */ + +#define fetch64_be_aligned(ptr) ((uint64_t)__builtin_e2k_ld_64s_be(ptr)) +#define fetch32_be_aligned(ptr) ((uint32_t)__builtin_e2k_ld_32u_be(ptr)) + +#endif /* __e2k__ Elbrus */ + +#elif defined(_MSC_VER) + +#if _MSC_FULL_VER < 190024234 && defined(_M_IX86) +#pragma message( \ + "For AES-NI at least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required.") +#endif +#if _MSC_FULL_VER < 191526730 +#pragma message( \ + "It is recommended to use \"Microsoft C/C++ Compiler\" version 19.15.26730 (Visual Studio 2017 15.8) or newer.") +#endif +#if _MSC_FULL_VER < 180040629 +#error At least "Microsoft C/C++ Compiler" version 18.00.40629 (Visual Studio 2013 Update 5) is required. +#endif + +#pragma warning(push, 1) + +#include +#include +#define likely(cond) (cond) +#define unlikely(cond) (cond) +#define unreachable() __assume(0) +#define bswap64(v) _byteswap_uint64(v) +#define bswap32(v) _byteswap_ulong(v) +#define bswap16(v) _byteswap_ushort(v) +#define rot64(v, s) _rotr64(v, s) +#define rot32(v, s) _rotr(v, s) +#define __always_inline __forceinline + +#if defined(_M_X64) || defined(_M_IA64) +#pragma intrinsic(_umul128) +#define mul_64x64_128(a, b, ph) _umul128(a, b, ph) +#pragma intrinsic(_addcarry_u64) +#define add64carry_first(base, addend, sum) _addcarry_u64(0, base, addend, sum) +#define add64carry_next(carry, base, addend, sum) \ + _addcarry_u64(carry, base, addend, sum) +#define add64carry_last(carry, base, addend, sum) \ + (void)_addcarry_u64(carry, base, addend, sum) +#endif + +#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) +#pragma intrinsic(__umulh) +#define mul_64x64_high(a, b) __umulh(a, b) +#endif + +#if defined(_M_IX86) +#pragma intrinsic(__emulu) +#define mul_32x32_64(a, b) __emulu(a, b) + +#if _MSC_VER >= 1915 /* LY: workaround for SSA-optimizer bug */ +#pragma intrinsic(_addcarry_u32) +#define add32carry_first(base, addend, sum) _addcarry_u32(0, base, addend, sum) +#define add32carry_next(carry, base, addend, sum) \ + _addcarry_u32(carry, base, addend, sum) +#define add32carry_last(carry, base, addend, sum) \ + (void)_addcarry_u32(carry, base, addend, sum) + +static __forceinline char +msvc32_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) { + uint32_t *const sum32 = (uint32_t *)sum; + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); + return add32carry_next(add32carry_first(base_32l, addend_32l, sum32), + base_32h, addend_32h, sum32 + 1); +} +#define add64carry_first(base, addend, sum) \ + msvc32_add64carry_first(base, addend, sum) + +static __forceinline char msvc32_add64carry_next(char carry, uint64_t base, + uint64_t addend, + uint64_t *sum) { + uint32_t *const sum32 = (uint32_t *)sum; + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); + return add32carry_next(add32carry_next(carry, base_32l, addend_32l, sum32), + base_32h, addend_32h, sum32 + 1); +} +#define add64carry_next(carry, base, addend, sum) \ + msvc32_add64carry_next(carry, base, addend, sum) + +static __forceinline void msvc32_add64carry_last(char carry, uint64_t base, + uint64_t addend, + uint64_t *sum) { + uint32_t *const sum32 = (uint32_t *)sum; + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); + add32carry_last(add32carry_next(carry, base_32l, addend_32l, sum32), base_32h, + addend_32h, sum32 + 1); +} +#define add64carry_last(carry, base, addend, sum) \ + msvc32_add64carry_last(carry, base, addend, sum) +#endif /* _MSC_FULL_VER >= 190024231 */ + +#elif defined(_M_ARM) +#define mul_32x32_64(a, b) _arm_umull(a, b) +#endif + +#pragma warning(pop) +#pragma warning(disable : 4514) /* 'xyz': unreferenced inline function \ + has been removed */ +#pragma warning(disable : 4710) /* 'xyz': function not inlined */ +#pragma warning(disable : 4711) /* function 'xyz' selected for \ + automatic inline expansion */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#pragma warning(disable : 4702) /* unreachable code */ +#endif /* Compiler */ + +#ifndef likely +#define likely(cond) (cond) +#endif +#ifndef unlikely +#define unlikely(cond) (cond) +#endif +#ifndef __maybe_unused +#define __maybe_unused +#endif +#ifndef __always_inline +#define __always_inline __inline +#endif +#ifndef unreachable +#define unreachable() \ + do { \ + } while (1) +#endif + +#ifndef bswap64 +#if defined(bswap_64) +#define bswap64 bswap_64 +#elif defined(__bswap_64) +#define bswap64 __bswap_64 +#else +static __always_inline uint64_t bswap64(uint64_t v) { + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | + ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | + ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | + ((v >> 40) & UINT64_C(0x000000000000ff00)); +} +#endif +#endif /* bswap64 */ + +#ifndef bswap32 +#if defined(bswap_32) +#define bswap32 bswap_32 +#elif defined(__bswap_32) +#define bswap32 __bswap_32 +#else +static __always_inline uint32_t bswap32(uint32_t v) { + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | + ((v >> 8) & UINT32_C(0x0000ff00)); +} +#endif +#endif /* bswap32 */ + +#ifndef bswap16 +#if defined(bswap_16) +#define bswap16 bswap_16 +#elif defined(__bswap_16) +#define bswap16 __bswap_16 +#else +static __always_inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; } +#endif +#endif /* bswap16 */ + +#if defined(__ia32__) || \ + T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT +/* The __builtin_assume_aligned() leads gcc/clang to load values into the + * registers, even when it is possible to directly use an operand from memory. + * This can lead to a shortage of registers and a significant slowdown. + * Therefore avoid unnecessary use of __builtin_assume_aligned() for x86. */ +#define read_unaligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) +#define read_aligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) +#endif /* __ia32__ */ + +#ifndef read_unaligned +#if defined(__GNUC__) || __has_attribute(__packed__) +typedef struct { + uint8_t unaligned_8; + uint16_t unaligned_16; + uint32_t unaligned_32; + uint64_t unaligned_64; +} __attribute__((__packed__)) t1ha_unaligned_proxy; +#define read_unaligned(ptr, bits) \ + (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ + t1ha_unaligned_proxy, unaligned_##bits))) \ + ->unaligned_##bits) +#elif defined(_MSC_VER) +#pragma warning( \ + disable : 4235) /* nonstandard extension used: '__unaligned' \ + * keyword not supported on this architecture */ +#define read_unaligned(ptr, bits) (*(const __unaligned uint##bits##_t *)(ptr)) +#else +#pragma pack(push, 1) +typedef struct { + uint8_t unaligned_8; + uint16_t unaligned_16; + uint32_t unaligned_32; + uint64_t unaligned_64; +} t1ha_unaligned_proxy; +#pragma pack(pop) +#define read_unaligned(ptr, bits) \ + (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ + t1ha_unaligned_proxy, unaligned_##bits))) \ + ->unaligned_##bits) +#endif +#endif /* read_unaligned */ + +#ifndef read_aligned +#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned) +#define read_aligned(ptr, bits) \ + (*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits)) +#elif (__GNUC_PREREQ(3, 3) || __has_attribute(__aligned__)) && \ + !defined(__clang__) +#define read_aligned(ptr, bits) \ + (*(const uint##bits##_t \ + __attribute__((__aligned__(ALIGNMENT_##bits))) *)(ptr)) +#elif __has_attribute(__assume_aligned__) + +static __always_inline const + uint16_t *__attribute__((__assume_aligned__(ALIGNMENT_16))) + cast_aligned_16(const void *ptr) { + return (const uint16_t *)ptr; +} +static __always_inline const + uint32_t *__attribute__((__assume_aligned__(ALIGNMENT_32))) + cast_aligned_32(const void *ptr) { + return (const uint32_t *)ptr; +} +static __always_inline const + uint64_t *__attribute__((__assume_aligned__(ALIGNMENT_64))) + cast_aligned_64(const void *ptr) { + return (const uint64_t *)ptr; +} + +#define read_aligned(ptr, bits) (*cast_aligned_##bits(ptr)) + +#elif defined(_MSC_VER) +#define read_aligned(ptr, bits) \ + (*(const __declspec(align(ALIGNMENT_##bits)) uint##bits##_t *)(ptr)) +#else +#define read_aligned(ptr, bits) (*(const uint##bits##_t *)(ptr)) +#endif +#endif /* read_aligned */ + +#ifndef prefetch +#if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \ + !defined(__ia32__) +#define prefetch(ptr) __builtin_prefetch(ptr) +#elif defined(_M_ARM64) || defined(_M_ARM) +#define prefetch(ptr) __prefetch(ptr) +#else +#define prefetch(ptr) \ + do { \ + (void)(ptr); \ + } while (0) +#endif +#endif /* prefetch */ + +#if __has_warning("-Wconstant-logical-operand") +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wconstant-logical-operand" +#elif defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wconstant-logical-operand" +#else +#pragma warning disable "constant-logical-operand" +#endif +#endif /* -Wconstant-logical-operand */ + +#if __has_warning("-Wtautological-pointer-compare") +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wtautological-pointer-compare" +#elif defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wtautological-pointer-compare" +#else +#pragma warning disable "tautological-pointer-compare" +#endif +#endif /* -Wtautological-pointer-compare */ + +/***************************************************************************/ + +#if __GNUC_PREREQ(4, 0) +#pragma GCC visibility push(hidden) +#endif /* __GNUC_PREREQ(4,0) */ + +/*---------------------------------------------------------- Little Endian */ + +#ifndef fetch16_le_aligned +static __maybe_unused __always_inline uint16_t +fetch16_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_aligned(v, 16); +#else + return bswap16(read_aligned(v, 16)); +#endif +} +#endif /* fetch16_le_aligned */ + +#ifndef fetch16_le_unaligned +static __maybe_unused __always_inline uint16_t +fetch16_le_unaligned(const void *v) { +#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + const uint8_t *p = (const uint8_t *)v; + return p[0] | (uint16_t)p[1] << 8; +#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_unaligned(v, 16); +#else + return bswap16(read_unaligned(v, 16)); +#endif +} +#endif /* fetch16_le_unaligned */ + +#ifndef fetch32_le_aligned +static __maybe_unused __always_inline uint32_t +fetch32_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_aligned(v, 32); +#else + return bswap32(read_aligned(v, 32)); +#endif +} +#endif /* fetch32_le_aligned */ + +#ifndef fetch32_le_unaligned +static __maybe_unused __always_inline uint32_t +fetch32_le_unaligned(const void *v) { +#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + return fetch16_le_unaligned(v) | + (uint32_t)fetch16_le_unaligned((const uint8_t *)v + 2) << 16; +#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_unaligned(v, 32); +#else + return bswap32(read_unaligned(v, 32)); +#endif +} +#endif /* fetch32_le_unaligned */ + +#ifndef fetch64_le_aligned +static __maybe_unused __always_inline uint64_t +fetch64_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_aligned(v, 64); +#else + return bswap64(read_aligned(v, 64)); +#endif +} +#endif /* fetch64_le_aligned */ + +#ifndef fetch64_le_unaligned +static __maybe_unused __always_inline uint64_t +fetch64_le_unaligned(const void *v) { +#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + return fetch32_le_unaligned(v) | + (uint64_t)fetch32_le_unaligned((const uint8_t *)v + 4) << 32; +#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return read_unaligned(v, 64); +#else + return bswap64(read_unaligned(v, 64)); +#endif +} +#endif /* fetch64_le_unaligned */ + +static __maybe_unused __always_inline uint64_t tail64_le_aligned(const void *v, + size_t tail) { + const uint8_t *const p = (const uint8_t *)v; +#if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__) + /* We can perform a 'oneshot' read, which is little bit faster. */ + const unsigned shift = ((8 - tail) & 7) << 3; + return fetch64_le_aligned(p) & ((~UINT64_C(0)) >> shift); +#else + uint64_t r = 0; + switch (tail & 7) { + default: + unreachable(); +/* fall through */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* For most CPUs this code is better when not needed byte reordering. */ + case 0: + return fetch64_le_aligned(p); + case 7: + r = (uint64_t)p[6] << 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 32; + /* fall through */ + case 4: + return r + fetch32_le_aligned(p); + case 3: + r = (uint64_t)p[2] << 16; + /* fall through */ + case 2: + return r + fetch16_le_aligned(p); + case 1: + return p[0]; +#else + case 0: + r = p[7] << 8; + /* fall through */ + case 7: + r += p[6]; + r <<= 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 8; + /* fall through */ + case 4: + r += p[3]; + r <<= 8; + /* fall through */ + case 3: + r += p[2]; + r <<= 8; + /* fall through */ + case 2: + r += p[1]; + r <<= 8; + /* fall through */ + case 1: + return r + p[0]; +#endif + } +#endif /* T1HA_USE_FAST_ONESHOT_READ */ +} + +#if T1HA_USE_FAST_ONESHOT_READ && \ + T1HA_SYS_UNALIGNED_ACCESS != T1HA_UNALIGNED_ACCESS__UNABLE && \ + defined(PAGESIZE) && PAGESIZE > 42 && !defined(__SANITIZE_ADDRESS__) +#define can_read_underside(ptr, size) \ + (((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0) +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +static __maybe_unused __always_inline uint64_t +tail64_le_unaligned(const void *v, size_t tail) { + const uint8_t *p = (const uint8_t *)v; +#if defined(can_read_underside) && \ + (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) + /* On some systems (e.g. x86_64) we can perform a 'oneshot' read, which + * is little bit faster. Thanks Marcin Żukowski + * for the reminder. */ + const unsigned offset = (8 - tail) & 7; + const unsigned shift = offset << 3; + if (likely(can_read_underside(p, 8))) { + p -= offset; + return fetch64_le_unaligned(p) >> shift; + } + return fetch64_le_unaligned(p) & ((~UINT64_C(0)) >> shift); +#else + uint64_t r = 0; + switch (tail & 7) { + default: + unreachable(); +/* fall through */ +#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* For most CPUs this code is better when not needed + * copying for alignment or byte reordering. */ + case 0: + return fetch64_le_unaligned(p); + case 7: + r = (uint64_t)p[6] << 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 32; + /* fall through */ + case 4: + return r + fetch32_le_unaligned(p); + case 3: + r = (uint64_t)p[2] << 16; + /* fall through */ + case 2: + return r + fetch16_le_unaligned(p); + case 1: + return p[0]; +#else + /* For most CPUs this code is better than a + * copying for alignment and/or byte reordering. */ + case 0: + r = p[7] << 8; + /* fall through */ + case 7: + r += p[6]; + r <<= 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 8; + /* fall through */ + case 4: + r += p[3]; + r <<= 8; + /* fall through */ + case 3: + r += p[2]; + r <<= 8; + /* fall through */ + case 2: + r += p[1]; + r <<= 8; + /* fall through */ + case 1: + return r + p[0]; +#endif + } +#endif /* can_read_underside */ +} + +/*------------------------------------------------------------- Big Endian */ + +#ifndef fetch16_be_aligned +static __maybe_unused __always_inline uint16_t +fetch16_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_aligned(v, 16); +#else + return bswap16(read_aligned(v, 16)); +#endif +} +#endif /* fetch16_be_aligned */ + +#ifndef fetch16_be_unaligned +static __maybe_unused __always_inline uint16_t +fetch16_be_unaligned(const void *v) { +#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + const uint8_t *p = (const uint8_t *)v; + return (uint16_t)p[0] << 8 | p[1]; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_unaligned(v, 16); +#else + return bswap16(read_unaligned(v, 16)); +#endif +} +#endif /* fetch16_be_unaligned */ + +#ifndef fetch32_be_aligned +static __maybe_unused __always_inline uint32_t +fetch32_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_aligned(v, 32); +#else + return bswap32(read_aligned(v, 32)); +#endif +} +#endif /* fetch32_be_aligned */ + +#ifndef fetch32_be_unaligned +static __maybe_unused __always_inline uint32_t +fetch32_be_unaligned(const void *v) { +#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + return (uint32_t)fetch16_be_unaligned(v) << 16 | + fetch16_be_unaligned((const uint8_t *)v + 2); +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_unaligned(v, 32); +#else + return bswap32(read_unaligned(v, 32)); +#endif +} +#endif /* fetch32_be_unaligned */ + +#ifndef fetch64_be_aligned +static __maybe_unused __always_inline uint64_t +fetch64_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_aligned(v, 64); +#else + return bswap64(read_aligned(v, 64)); +#endif +} +#endif /* fetch64_be_aligned */ + +#ifndef fetch64_be_unaligned +static __maybe_unused __always_inline uint64_t +fetch64_be_unaligned(const void *v) { +#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + return (uint64_t)fetch32_be_unaligned(v) << 32 | + fetch32_be_unaligned((const uint8_t *)v + 4); +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return read_unaligned(v, 64); +#else + return bswap64(read_unaligned(v, 64)); +#endif +} +#endif /* fetch64_be_unaligned */ + +static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, + size_t tail) { + const uint8_t *const p = (const uint8_t *)v; +#if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__) + /* We can perform a 'oneshot' read, which is little bit faster. */ + const unsigned shift = ((8 - tail) & 7) << 3; + return fetch64_be_aligned(p) >> shift; +#else + switch (tail & 7) { + default: + unreachable(); +/* fall through */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + /* For most CPUs this code is better when not byte reordering. */ + case 1: + return p[0]; + case 2: + return fetch16_be_aligned(p); + case 3: + return (uint32_t)fetch16_be_aligned(p) << 8 | p[2]; + case 4: + return fetch32_be_aligned(p); + case 5: + return (uint64_t)fetch32_be_aligned(p) << 8 | p[4]; + case 6: + return (uint64_t)fetch32_be_aligned(p) << 16 | fetch16_be_aligned(p + 4); + case 7: + return (uint64_t)fetch32_be_aligned(p) << 24 | + (uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6]; + case 0: + return fetch64_be_aligned(p); +#else + case 1: + return p[0]; + case 2: + return p[1] | (uint32_t)p[0] << 8; + case 3: + return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; + case 4: + return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | + (uint32_t)p[0] << 24; + case 5: + return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | + (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; + case 6: + return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | + (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; + case 7: + return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | + (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 | + (uint64_t)p[0] << 48; + case 0: + return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | + (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 | + (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; +#endif + } +#endif /* T1HA_USE_FAST_ONESHOT_READ */ +} + +static __maybe_unused __always_inline uint64_t +tail64_be_unaligned(const void *v, size_t tail) { + const uint8_t *p = (const uint8_t *)v; +#if defined(can_read_underside) && \ + (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) + /* On some systems (e.g. x86_64) we can perform a 'oneshot' read, which + * is little bit faster. Thanks Marcin Żukowski + * for the reminder. */ + const unsigned offset = (8 - tail) & 7; + const unsigned shift = offset << 3; + if (likely(can_read_underside(p, 8))) { + p -= offset; + return fetch64_be_unaligned(p) & ((~UINT64_C(0)) >> shift); + } + return fetch64_be_unaligned(p) >> shift; +#else + switch (tail & 7) { + default: + unreachable(); +/* fall through */ +#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + /* For most CPUs this code is better when not needed + * copying for alignment or byte reordering. */ + case 1: + return p[0]; + case 2: + return fetch16_be_unaligned(p); + case 3: + return (uint32_t)fetch16_be_unaligned(p) << 8 | p[2]; + case 4: + return fetch32_be(p); + case 5: + return (uint64_t)fetch32_be_unaligned(p) << 8 | p[4]; + case 6: + return (uint64_t)fetch32_be_unaligned(p) << 16 | + fetch16_be_unaligned(p + 4); + case 7: + return (uint64_t)fetch32_be_unaligned(p) << 24 | + (uint32_t)fetch16_be_unaligned(p + 4) << 8 | p[6]; + case 0: + return fetch64_be_unaligned(p); +#else + /* For most CPUs this code is better than a + * copying for alignment and/or byte reordering. */ + case 1: + return p[0]; + case 2: + return p[1] | (uint32_t)p[0] << 8; + case 3: + return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; + case 4: + return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | + (uint32_t)p[0] << 24; + case 5: + return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | + (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; + case 6: + return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | + (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; + case 7: + return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | + (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 | + (uint64_t)p[0] << 48; + case 0: + return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | + (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 | + (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; +#endif + } +#endif /* can_read_underside */ +} + +/***************************************************************************/ + +#ifndef rot64 +static __maybe_unused __always_inline uint64_t rot64(uint64_t v, unsigned s) { + return (v >> s) | (v << (64 - s)); +} +#endif /* rot64 */ + +#ifndef mul_32x32_64 +static __maybe_unused __always_inline uint64_t mul_32x32_64(uint32_t a, + uint32_t b) { + return a * (uint64_t)b; +} +#endif /* mul_32x32_64 */ + +#ifndef add64carry_first +static __maybe_unused __always_inline unsigned +add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) { +#if __has_builtin(__builtin_addcll) + unsigned long long carryout; + *sum = __builtin_addcll(base, addend, 0, &carryout); + return (unsigned)carryout; +#else + *sum = base + addend; + return *sum < addend; +#endif /* __has_builtin(__builtin_addcll) */ +} +#endif /* add64carry_fist */ + +#ifndef add64carry_next +static __maybe_unused __always_inline unsigned +add64carry_next(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) { +#if __has_builtin(__builtin_addcll) + unsigned long long carryout; + *sum = __builtin_addcll(base, addend, carry, &carryout); + return (unsigned)carryout; +#else + *sum = base + addend + carry; + return *sum < addend || (carry && *sum == addend); +#endif /* __has_builtin(__builtin_addcll) */ +} +#endif /* add64carry_next */ + +#ifndef add64carry_last +static __maybe_unused __always_inline void +add64carry_last(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) { +#if __has_builtin(__builtin_addcll) + unsigned long long carryout; + *sum = __builtin_addcll(base, addend, carry, &carryout); + (void)carryout; +#else + *sum = base + addend + carry; +#endif /* __has_builtin(__builtin_addcll) */ +} +#endif /* add64carry_last */ + +#ifndef mul_64x64_128 +static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a, + uint64_t b, + uint64_t *h) { +#if (defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) && \ + (!defined(__LCC__) || __LCC__ != 124) + __uint128_t r = (__uint128_t)a * (__uint128_t)b; + /* modern GCC could nicely optimize this */ + *h = (uint64_t)(r >> 64); + return (uint64_t)r; +#elif defined(mul_64x64_high) + *h = mul_64x64_high(a, b); + return a * b; +#else + /* performs 64x64 to 128 bit multiplication */ + const uint64_t ll = mul_32x32_64((uint32_t)a, (uint32_t)b); + const uint64_t lh = mul_32x32_64(a >> 32, (uint32_t)b); + const uint64_t hl = mul_32x32_64((uint32_t)a, b >> 32); + const uint64_t hh = mul_32x32_64(a >> 32, b >> 32); + + /* Few simplification are possible here for 32-bit architectures, + * but thus we would lost compatibility with the original 64-bit + * version. Think is very bad idea, because then 32-bit t1ha will + * still (relatively) very slowly and well yet not compatible. */ + uint64_t l; + add64carry_last(add64carry_first(ll, lh << 32, &l), hh, lh >> 32, h); + add64carry_last(add64carry_first(l, hl << 32, &l), *h, hl >> 32, h); + return l; +#endif +} +#endif /* mul_64x64_128() */ + +#ifndef mul_64x64_high +static __maybe_unused __always_inline uint64_t mul_64x64_high(uint64_t a, + uint64_t b) { + uint64_t h; + mul_64x64_128(a, b, &h); + return h; +} +#endif /* mul_64x64_high */ + +/***************************************************************************/ + +/* 'magic' primes */ +static const uint64_t prime_0 = UINT64_C(0xEC99BF0D8372CAAB); +static const uint64_t prime_1 = UINT64_C(0x82434FE90EDCEF39); +static const uint64_t prime_2 = UINT64_C(0xD4F06DB99D67BE4B); +static const uint64_t prime_3 = UINT64_C(0xBD9CACC22C6E9571); +static const uint64_t prime_4 = UINT64_C(0x9C06FAF4D023E3AB); +static const uint64_t prime_5 = UINT64_C(0xC060724A8424F345); +static const uint64_t prime_6 = UINT64_C(0xCB5AF53AE3AAAC31); + +/* xor high and low parts of full 128-bit product */ +static __maybe_unused __always_inline uint64_t mux64(uint64_t v, + uint64_t prime) { + uint64_t l, h; + l = mul_64x64_128(v, prime, &h); + return l ^ h; +} + +static __maybe_unused __always_inline uint64_t final64(uint64_t a, uint64_t b) { + uint64_t x = (a + rot64(b, 41)) * prime_0; + uint64_t y = (rot64(a, 23) + b) * prime_6; + return mux64(x ^ y, prime_5); +} + +static __maybe_unused __always_inline void mixup64(uint64_t *__restrict a, + uint64_t *__restrict b, + uint64_t v, uint64_t prime) { + uint64_t h; + *a ^= mul_64x64_128(*b + v, prime, &h); + *b += h; +} + +/***************************************************************************/ + +typedef union t1ha_uint128 { +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + __uint128_t v; +#endif + struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + uint64_t l, h; +#else + uint64_t h, l; +#endif + }; +} t1ha_uint128_t; + +static __maybe_unused __always_inline t1ha_uint128_t +not128(const t1ha_uint128_t v) { + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = ~v.v; +#else + r.l = ~v.l; + r.h = ~v.h; +#endif + return r; +} + +static __maybe_unused __always_inline t1ha_uint128_t +left128(const t1ha_uint128_t v, unsigned s) { + t1ha_uint128_t r; + assert(s < 128); +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = v.v << s; +#else + r.l = (s < 64) ? v.l << s : 0; + r.h = (s < 64) ? (v.h << s) | (s ? v.l >> (64 - s) : 0) : v.l << (s - 64); +#endif + return r; +} + +static __maybe_unused __always_inline t1ha_uint128_t +right128(const t1ha_uint128_t v, unsigned s) { + t1ha_uint128_t r; + assert(s < 128); +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = v.v >> s; +#else + r.l = (s < 64) ? (s ? v.h << (64 - s) : 0) | (v.l >> s) : v.h >> (s - 64); + r.h = (s < 64) ? v.h >> s : 0; +#endif + return r; +} + +static __maybe_unused __always_inline t1ha_uint128_t or128(t1ha_uint128_t x, + t1ha_uint128_t y) { + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = x.v | y.v; +#else + r.l = x.l | y.l; + r.h = x.h | y.h; +#endif + return r; +} + +static __maybe_unused __always_inline t1ha_uint128_t xor128(t1ha_uint128_t x, + t1ha_uint128_t y) { + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = x.v ^ y.v; +#else + r.l = x.l ^ y.l; + r.h = x.h ^ y.h; +#endif + return r; +} + +static __maybe_unused __always_inline t1ha_uint128_t rot128(t1ha_uint128_t v, + unsigned s) { + s &= 127; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + v.v = (v.v << (128 - s)) | (v.v >> s); + return v; +#else + return s ? or128(left128(v, 128 - s), right128(v, s)) : v; +#endif +} + +static __maybe_unused __always_inline t1ha_uint128_t add128(t1ha_uint128_t x, + t1ha_uint128_t y) { + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = x.v + y.v; +#else + add64carry_last(add64carry_first(x.l, y.l, &r.l), x.h, y.h, &r.h); +#endif + return r; +} + +static __maybe_unused __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x, + t1ha_uint128_t y) { + t1ha_uint128_t r; +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + r.v = x.v * y.v; +#else + r.l = mul_64x64_128(x.l, y.l, &r.h); + r.h += x.l * y.h + y.l * x.h; +#endif + return r; +} + +/***************************************************************************/ + +#if T1HA0_AESNI_AVAILABLE && defined(__ia32__) +uint64_t t1ha_ia32cpu_features(void); + +static __maybe_unused __always_inline bool +t1ha_ia32_AESNI_avail(uint64_t ia32cpu_features) { + /* check for AES-NI */ + return (ia32cpu_features & UINT32_C(0x02000000)) != 0; +} + +static __maybe_unused __always_inline bool +t1ha_ia32_AVX_avail(uint64_t ia32cpu_features) { + /* check for any AVX */ + return (ia32cpu_features & UINT32_C(0x1A000000)) == UINT32_C(0x1A000000); +} + +static __maybe_unused __always_inline bool +t1ha_ia32_AVX2_avail(uint64_t ia32cpu_features) { + /* check for 'Advanced Vector Extensions 2' */ + return ((ia32cpu_features >> 32) & 32) != 0; +} + +#endif /* T1HA0_AESNI_AVAILABLE && __ia32__ */ diff --git a/include/t1ha_selfcheck.h b/include/t1ha_selfcheck.h new file mode 100644 index 00000000..ff7c589c --- /dev/null +++ b/include/t1ha_selfcheck.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2016-2020 Positive Technologies, https://www.ptsecurity.com, + * Fast Positive Hash. + * + * Portions Copyright (c) 2010-2020 Leonid Yuriev , + * The 1Hippeus project (t1h). + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgement in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * t1ha = { Fast Positive Hash, aka "Позитивный Хэш" } + * by [Positive Technologies](https://www.ptsecurity.ru) + * + * Briefly, it is a 64-bit Hash Function: + * 1. Created for 64-bit little-endian platforms, in predominantly for x86_64, + * but portable and without penalties it can run on any 64-bit CPU. + * 2. In most cases up to 15% faster than City64, xxHash, mum-hash, metro-hash + * and all others portable hash-functions (which do not use specific + * hardware tricks). + * 3. Not suitable for cryptography. + * + * The Future will (be) Positive. Всё будет хорошо. + * + * ACKNOWLEDGEMENT: + * The t1ha was originally developed by Leonid Yuriev (Леонид Юрьев) + * for The 1Hippeus project - zerocopy messaging in the spirit of Sparta! + */ + +#pragma once +#if defined(_MSC_VER) && _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif /* MSVC */ +#include "t1ha.h" + +/***************************************************************************/ +/* Self-checking */ + +extern const uint8_t t1ha_test_pattern[64]; +int t1ha_selfcheck(uint64_t (*hash)(const void *, size_t, uint64_t), + const uint64_t *reference_values); + +#ifndef T1HA2_DISABLED +extern const uint64_t t1ha_refval_2atonce[81]; +extern const uint64_t t1ha_refval_2atonce128[81]; +extern const uint64_t t1ha_refval_2stream[81]; +extern const uint64_t t1ha_refval_2stream128[81]; +#endif /* T1HA2_DISABLED */ + +#ifndef T1HA1_DISABLED +extern const uint64_t t1ha_refval_64le[81]; +extern const uint64_t t1ha_refval_64be[81]; +#endif /* T1HA1_DISABLED */ + +#ifndef T1HA0_DISABLED +extern const uint64_t t1ha_refval_32le[81]; +extern const uint64_t t1ha_refval_32be[81]; +#if T1HA0_AESNI_AVAILABLE +extern const uint64_t t1ha_refval_ia32aes_a[81]; +extern const uint64_t t1ha_refval_ia32aes_b[81]; +#endif /* T1HA0_AESNI_AVAILABLE */ +#endif /* T1HA0_DISABLED */ diff --git a/include/xxhash.h b/include/xxhash.h index 9a880470..d11f0f63 100644 --- a/include/xxhash.h +++ b/include/xxhash.h @@ -1,7 +1,7 @@ /* * xxHash - Extremely Fast Hash algorithm * Header File - * Copyright (C) 2012-2024 Yann Collet + * Copyright (C) 2012-2023 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * @@ -32,328 +32,553 @@ * - xxHash homepage: https://www.xxhash.com * - xxHash source repository: https://github.com/Cyan4973/xxHash */ + /*! * @mainpage xxHash * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * * @file xxhash.h * xxHash prototypes and implementation */ -/* TODO: update */ -/* Notice extracted from xxHash homepage: - -xxHash is an extremely fast hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo -@3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MurmurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -Note: SMHasher's CRC32 implementation is not the fastest one. -Other speed-oriented implementations can be faster, -especially in combination with PCLMUL instruction: -https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 - -A 64-bit version, named XXH64, is available since r35. -It offers much better speed, but for 64-bit applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ - -#if defined(__cplusplus) -extern "C" { +#if defined (__cplusplus) +extern "C" { #endif /* **************************** * INLINE mode ******************************/ /*! - * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * * Use these build macros to inline xxhash into the target unit. * Inlining improves performance on small inputs, especially when the length is * expressed as a compile-time constant: * - * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html * * It also keeps xxHash symbols private to the unit, so they are not exported. * * Usage: + * @code{.c} * #define XXH_INLINE_ALL * #include "xxhash.h" - * + * @endcode * Do not compile and link xxhash.o as a separate object, as it is not useful. */ -#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) && \ - !defined(XXH_INLINE_ALL_31684351384) -/* this section should be traversed only once */ - #define XXH_INLINE_ALL_31684351384 -/* give access to the advanced API, required to compile implementations */ - #undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ - #define XXH_STATIC_LINKING_ONLY -/* make all functions private */ - #undef XXH_PUBLIC_API - #if defined(__GNUC__) - #define XXH_PUBLIC_API static __inline __attribute__((unused)) - #elif defined(__cplusplus) || \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) - #define XXH_PUBLIC_API static inline - #elif defined(_MSC_VER) - #define XXH_PUBLIC_API static __inline - #else - /* note: this version may generate warnings for unused static functions */ - #define XXH_PUBLIC_API static - #endif +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif -/* - * This part deals with the special case where a unit wants to inline xxHash, - * but "xxhash.h" has previously been included without XXH_INLINE_ALL, - * such as part of some previously included *.h header file. - * Without further action, the new include would just be ignored, - * and functions would effectively _not_ be inlined (silent failure). - * The following macros solve this situation by prefixing all inlined names, - * avoiding naming collision with previous inclusions. - */ -/* Before that, we unconditionally #undef all symbols, - * in case they were already defined with XXH_NAMESPACE. - * They will then be redefined for XXH_INLINE_ALL - */ - #undef XXH_versionNumber -/* XXH32 */ - #undef XXH32 - #undef XXH32_createState - #undef XXH32_freeState - #undef XXH32_reset - #undef XXH32_update - #undef XXH32_digest - #undef XXH32_copyState - #undef XXH32_canonicalFromHash - #undef XXH32_hashFromCanonical -/* XXH64 */ - #undef XXH64 - #undef XXH64_createState - #undef XXH64_freeState - #undef XXH64_reset - #undef XXH64_update - #undef XXH64_digest - #undef XXH64_copyState - #undef XXH64_canonicalFromHash - #undef XXH64_hashFromCanonical -/* XXH3_64bits */ - #undef XXH3_64bits - #undef XXH3_64bits_withSecret - #undef XXH3_64bits_withSeed - #undef XXH3_createState - #undef XXH3_freeState - #undef XXH3_copyState - #undef XXH3_64bits_reset - #undef XXH3_64bits_reset_withSeed - #undef XXH3_64bits_reset_withSecret - #undef XXH3_64bits_update - #undef XXH3_64bits_digest - #undef XXH3_generateSecret -/* XXH3_128bits */ - #undef XXH128 - #undef XXH3_128bits - #undef XXH3_128bits_withSeed - #undef XXH3_128bits_withSecret - #undef XXH3_128bits_reset - #undef XXH3_128bits_reset_withSeed - #undef XXH3_128bits_reset_withSecret - #undef XXH3_128bits_update - #undef XXH3_128bits_digest - #undef XXH128_isEqual - #undef XXH128_cmp - #undef XXH128_canonicalFromHash - #undef XXH128_hashFromCanonical -/* Finally, free the namespace itself */ - #undef XXH_NAMESPACE - -/* employ the namespace for XXH_INLINE_ALL */ - #define XXH_NAMESPACE XXH_INLINE_ -/* - * Some identifiers (enums, type names) are not symbols, - * but they must nonetheless be renamed to avoid redeclaration. - * Alternative solution: do not redeclare them. - * However, this requires some #ifdefs, and has a more dispersed impact. - * Meanwhile, renaming can be achieved in a single place. - */ - #define XXH_IPREF(Id) XXH_NAMESPACE##Id - #define XXH_OK XXH_IPREF(XXH_OK) - #define XXH_ERROR XXH_IPREF(XXH_ERROR) - #define XXH_errorcode XXH_IPREF(XXH_errorcode) - #define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) - #define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) - #define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) - #define XXH32_state_s XXH_IPREF(XXH32_state_s) - #define XXH32_state_t XXH_IPREF(XXH32_state_t) - #define XXH64_state_s XXH_IPREF(XXH64_state_s) - #define XXH64_state_t XXH_IPREF(XXH64_state_t) - #define XXH3_state_s XXH_IPREF(XXH3_state_s) - #define XXH3_state_t XXH_IPREF(XXH3_state_t) - #define XXH128_hash_t XXH_IPREF(XXH128_hash_t) -/* Ensure the header is parsed again, even if it was previously included */ - #undef XXHASH_H_5627135585666179 - #undef XXHASH_H_STATIC_13879238742 -#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ /* **************************************************************** * Stable API *****************************************************************/ #ifndef XXHASH_H_5627135585666179 - #define XXHASH_H_5627135585666179 1 - - /*! - * @defgroup public Public API - * Contains details on the public xxHash functions. - * @{ - - */ - /* specific declaration modes for Windows */ - #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) - #if defined(WIN32) && defined(_MSC_VER) && \ - (defined(XXH_IMPORT) || defined(XXH_EXPORT)) - #ifdef XXH_EXPORT - #define XXH_PUBLIC_API __declspec(dllexport) - #elif XXH_IMPORT - #define XXH_PUBLIC_API __declspec(dllimport) - #endif - #else - #define XXH_PUBLIC_API /* do nothing */ - #endif - #endif - - #ifdef XXH_DOXYGEN - /*! - * @brief Emulate a namespace by transparently prefixing all symbols. - * - * If you want to include _and expose_ xxHash functions from within your own - * library, but also want to avoid symbol collisions with other libraries - * which may also include xxHash, you can use XXH_NAMESPACE to automatically - * prefix any public symbol from xxhash library with the value of - * XXH_NAMESPACE (therefore, avoid empty or numeric values). - * - * Note that no change is required within the calling program as long as it - * includes `xxhash.h`: Regular symbol names will be automatically - * translated by this header. - */ - #define XXH_NAMESPACE /* YOUR NAME HERE */ - #undef XXH_NAMESPACE - #endif - - #ifdef XXH_NAMESPACE - #define XXH_CAT(A, B) A##B - #define XXH_NAME2(A, B) XXH_CAT(A, B) - #define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) - /* XXH32 */ - #define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) - #define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) - #define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) - #define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) - #define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) - #define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) - #define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) - #define XXH32_canonicalFromHash \ - XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) - #define XXH32_hashFromCanonical \ - XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) - /* XXH64 */ - #define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) - #define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) - #define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) - #define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) - #define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) - #define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) - #define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) - #define XXH64_canonicalFromHash \ - XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) - #define XXH64_hashFromCanonical \ - XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) - /* XXH3_64bits */ - #define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) - #define XXH3_64bits_withSecret \ - XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) - #define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) - #define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) - #define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) - #define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) - #define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) - #define XXH3_64bits_reset_withSeed \ - XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) - #define XXH3_64bits_reset_withSecret \ - XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) - #define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) - #define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) - #define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) - /* XXH3_128bits */ - #define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) - #define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) - #define XXH3_128bits_withSeed \ - XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) - #define XXH3_128bits_withSecret \ - XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) - #define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) - #define XXH3_128bits_reset_withSeed \ - XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) - #define XXH3_128bits_reset_withSecret \ - XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) - #define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) - #define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) - #define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) - #define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) - #define XXH128_canonicalFromHash \ - XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) - #define XXH128_hashFromCanonical \ - XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) - #endif - - /* ************************************* - * Version - ***************************************/ - #define XXH_VERSION_MAJOR 0 - #define XXH_VERSION_MINOR 8 - #define XXH_VERSION_RELEASE 1 - #define XXH_VERSION_NUMBER \ - (XXH_VERSION_MAJOR * 100 * 100 + XXH_VERSION_MINOR * 100 + \ - XXH_VERSION_RELEASE) +#define XXHASH_H_5627135585666179 1 + +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) /*! * @brief Obtains the xxHash version. * - * This is only useful when xxHash is compiled as a shared library, as it is - * independent of the version defined in the header. + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. * - * @return `XXH_VERSION_NUMBER` as of when the libray was compiled. + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +/*! + * @brief Exit code for the streaming API. */ -XXH_PUBLIC_API unsigned XXH_versionNumber(void); +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; - /* **************************** - * Definitions - ******************************/ - #include /* size_t */ -typedef enum { XXH_OK = 0, XXH_ERROR } XXH_errorcode; - /*-********************************************************************** - * 32-bit hash - ************************************************************************/ - #if defined(XXH_DOXYGEN) /* Don't show include */ +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ /*! * @brief An unsigned 32-bit integer. * @@ -361,51 +586,44 @@ typedef enum { XXH_OK = 0, XXH_ERROR } XXH_errorcode; */ typedef uint32_t XXH32_hash_t; - #elif !defined(__VMS) && \ - (defined(__cplusplus) || \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) - #include -typedef uint32_t XXH32_hash_t; - - #else - #include - #if UINT_MAX == 0xFFFFFFFFUL -typedef unsigned int XXH32_hash_t; - #else - #if ULONG_MAX == 0xFFFFFFFFUL -typedef unsigned long XXH32_hash_t; - #else - #error "unsupported platform: need a 32-bit type" - #endif - #endif - #endif +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif /*! * @} * - * @defgroup xxh32_family XXH32 family + * @defgroup XXH32_family XXH32 family * @ingroup public * Contains functions used in the classic 32-bit xxHash algorithm. * * @note - * XXH32 is considered rather weak by today's standards. - * The @ref xxh3_family provides competitive speed for both 32-bit and 64-bit - * systems, and offers true 64/128 bit hash results. It provides a superior - * level of dispersion, and greatly reduces the risks of collisions. + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. * - * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families - * @see @ref xxh32_impl for implementation details + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details * @{ - */ /*! * @brief Calculates the 32-bit hash of @p input using xxHash32. * - * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s - * - * @param input The block of data to be hashed, at least @p length bytes in - * size. + * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * @param seed The 32-bit seed to alter the hash's output predictably. * @@ -414,94 +632,46 @@ typedef unsigned long XXH32_hash_t; * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * @return The calculated 32-bit hash value. - * - * @see - * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): - * Direct equivalents for the other variants of xxHash. - * @see - * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. - */ -XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t length, - XXH32_hash_t seed); - -/*! - * Streaming functions generate the xxHash value from an incremental input. - * This method is slower than single-call functions, due to state management. - * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. - * - * An XXH state must first be allocated using `XXH*_createState()`. - * - * Start a new hash by initializing the state with a seed using `XXH*_reset()`. - * - * Then, feed the hash state by calling `XXH*_update()` as many times as - * necessary. - * - * The function returns an error code, with 0 meaning OK, and any other value - * meaning there is an error. - * - * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. - * This function returns the nn-bits hash as an int or long long. - * - * It's still possible to continue inserting input into the hash state after a - * digest, and generate new hash values later on by invoking `XXH*_digest()`. - * - * When done, release the state using `XXH*_freeState()`. - * - * Example code for incrementally hashing a file: - * @code{.c} - * #include - * #include - * #define BUFFER_SIZE 256 - * - * // Note: XXH64 and XXH3 use the same interface. - * XXH32_hash_t - * hashFile(FILE* stream) - * { - - * XXH32_state_t* state; - * unsigned char buf[BUFFER_SIZE]; - * size_t amt; - * XXH32_hash_t hash; + * @return The calculated 32-bit xxHash32 value. * - * state = XXH32_createState(); // Create a state - * assert(state != NULL); // Error check here - * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed - * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { - - * XXH32_update(state, buf, amt); // Hash the file in chunks - * } - * hash = XXH32_digest(state); // Finalize the hash - * XXH32_freeState(state); // Clean up - * return hash; - * } - * @endcode + * @see @ref single_shot_example "Single Shot Example" for an example. */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); +#ifndef XXH_NO_STREAM /*! * @typedef struct XXH32_state_s XXH32_state_t * @brief The opaque state struct for the XXH32 streaming API. * * @see XXH32_state_s for details. + * @see @ref streaming_example "Streaming Example" */ typedef struct XXH32_state_s XXH32_state_t; /*! * @brief Allocates an @ref XXH32_state_t. * - * Must be freed with XXH32_freeState(). - * @return An allocated XXH32_state_t on success, `NULL` on failure. + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + * + * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH32_state_t *XXH32_createState(void); +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); /*! * @brief Frees an @ref XXH32_state_t. * - * Must be allocated with XXH32_createState(). - * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref - * XXH32_createState(). - * @return XXH_OK. + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + * @see @ref streaming_example "Streaming Example" + * */ -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); /*! * @brief Copies one @ref XXH32_state_t to another. * @@ -510,33 +680,31 @@ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr); * @pre * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t *dst_state, - const XXH32_state_t *src_state); +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); /*! * @brief Resets an @ref XXH32_state_t to begin a new hash. * - * This function resets and seeds a state. Call it before @ref XXH32_update(). - * * @param statePtr The state struct to reset. * @param seed The 32-bit seed to alter the hash result predictably. * * @pre * @p statePtr must not be `NULL`. * - * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, - XXH32_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); /*! * @brief Consumes a block of @p input to an @ref XXH32_state_t. * - * Call this to incrementally consume blocks of data. - * * @param statePtr The state struct to update. - * @param input The block of data to be hashed, at least @p length bytes in - * size. + * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * * @pre @@ -546,70 +714,55 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *statePtr, - const void *input, size_t length); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); /*! * @brief Returns the calculated hash value from an @ref XXH32_state_t. * - * @note - * Calling XXH32_digest() will not affect @p statePtr, so you can update, - * digest, and update again. - * * @param statePtr The state struct to calculate the hash from. * * @pre * @p statePtr must not be `NULL`. * - * @return The calculated xxHash32 value from that state. - */ -XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *statePtr); - -/******* Canonical representation *******/ - -/* - * The default return values from XXH functions are unsigned 32 and 64 bit - * integers. - * This the simplest and fastest format for further post-processing. - * - * However, this leaves open the question of what is the order on the byte - * level, since little and big endian conventions will store the same number - * differently. - * - * The canonical representation settles this issue by mandating big-endian - * convention, the same convention as human-readable numbers (large digits - * first). + * @return The calculated 32-bit xxHash32 value from that state. * - * When writing hash values to storage, sending them over a network, or printing - * them, it's highly recommended to use the canonical representation to ensure - * portability across a wider range of systems, present and future. + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. * - * The following functions allow transformation of hash values to and from - * canonical format. + * @see @ref streaming_example "Streaming Example" */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ /*! * @brief Canonical (big endian) representation of @ref XXH32_hash_t. */ typedef struct { - - unsigned char digest[4]; /*!< Hash bytes, big endian */ - + unsigned char digest[4]; /*!< Hash bytes, big endian */ } XXH32_canonical_t; /*! * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. * - * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param dst The @ref XXH32_canonical_t pointer to be stored to. * @param hash The @ref XXH32_hash_t to be converted. * * @pre * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" */ -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, - XXH32_hash_t hash); +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); /*! * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. @@ -620,103 +773,127 @@ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, * @p src must not be `NULL`. * * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. */ -XXH_PUBLIC_API XXH32_hash_t -XXH32_hashFromCanonical(const XXH32_canonical_t *src); - - #ifdef __has_attribute - #define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) - #else - #define XXH_HAS_ATTRIBUTE(x) 0 - #endif - - /* C-language Attributes are added in C23. */ - #if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && \ - defined(__has_c_attribute) - #define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) - #else - #define XXH_HAS_C_ATTRIBUTE(x) 0 - #endif - - #if defined(__cplusplus) && defined(__has_cpp_attribute) - #define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) - #else - #define XXH_HAS_CPP_ATTRIBUTE(x) 0 - #endif - - /* - Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' - attribute introduced in CPP17 and C23. CPP17 : - https://en.cppreference.com/w/cpp/language/attributes/fallthrough C23 : - https://en.cppreference.com/w/c/language/attributes/fallthrough - */ - #if XXH_HAS_C_ATTRIBUTE(x) - #define XXH_FALLTHROUGH [[fallthrough]] - #elif XXH_HAS_CPP_ATTRIBUTE(x) - #define XXH_FALLTHROUGH [[fallthrough]] - #elif XXH_HAS_ATTRIBUTE(__fallthrough__) - #define XXH_FALLTHROUGH __attribute__((fallthrough)) - #else - #define XXH_FALLTHROUGH - #endif +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ + /*! * @} * @ingroup public * @{ - */ - #ifndef XXH_NO_LONG_LONG - /*-********************************************************************** - * 64-bit hash - ************************************************************************/ - #if defined(XXH_DOXYGEN) /* don't include */ +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ /*! * @brief An unsigned 64-bit integer. * * Not necessarily defined to `uint64_t` but functionally equivalent. */ typedef uint64_t XXH64_hash_t; - #elif !defined(__VMS) && \ - (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ - (__STDC_VERSION__ >= 199901L) /* C99 */)) - #include -typedef uint64_t XXH64_hash_t; - #else - #include - #if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL -/* LP64 ABI says uint64_t is unsigned long */ -typedef unsigned long XXH64_hash_t; - #else -/* the following type must have a width of 64-bit */ -typedef unsigned long long XXH64_hash_t; - #endif - #endif +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif /*! * @} * - * @defgroup xxh64_family XXH64 family + * @defgroup XXH64_family XXH64 family * @ingroup public * @{ - * Contains functions used in the classic 64-bit xxHash algorithm. * * @note * XXH3 provides competitive speed for both 32-bit and 64-bit systems, - * and offers true 64/128 bit hash results. It provides a superior level of - * dispersion, and greatly reduces the risks of collisions. + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. */ /*! * @brief Calculates the 64-bit hash of @p input using xxHash64. * - * This function usually runs faster on 64-bit systems, but slower on 32-bit - * systems (see benchmark). - * - * @param input The block of data to be hashed, at least @p length bytes in - * size. + * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * @param seed The 64-bit seed to alter the hash's output predictably. * @@ -725,62 +902,166 @@ typedef unsigned long long XXH64_hash_t; * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * @return The calculated 64-bit hash. + * @return The calculated 64-bit xxHash64 value. * - * @see - * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): - * Direct equivalents for the other variants of xxHash. - * @see - * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t length, - XXH64_hash_t seed); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); /******* Streaming *******/ +#ifndef XXH_NO_STREAM /*! * @brief The opaque state struct for the XXH64 streaming API. * * @see XXH64_state_s for details. + * @see @ref streaming_example "Streaming Example" */ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH64_state_t *XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t *dst_state, - const XXH64_state_t *src_state); - -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t *statePtr, - XXH64_hash_t seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *statePtr, - const void *input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t *statePtr); - -/******* Canonical representation *******/ -typedef struct { - - unsigned char digest[sizeof(XXH64_hash_t)]; - -} XXH64_canonical_t; - -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, - XXH64_hash_t hash); -XXH_PUBLIC_API XXH64_hash_t -XXH64_hashFromCanonical(const XXH64_canonical_t *src); +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ /*! - * @} - * ************************************************************************ - * @defgroup xxh3_family XXH3 family - * @ingroup public - * @{ - + * @brief Allocates an @ref XXH64_state_t. * - * XXH3 is a more recent hash algorithm featuring: - * - Improved speed for both small and large inputs - * - True 64-bit and 128-bit outputs - * - SIMD acceleration - * - Improved 32-bit viability + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. * - * Speed analysis methodology is explained here: + * @note Must be freed with XXH64_freeState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; + +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 + +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: * * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html * @@ -790,16 +1071,26 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src); * * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, * but does not require it. - * Any 32-bit and 64-bit targets that can run XXH32 smoothly - * can run XXH3 at competitive speeds, even without vector support. - * Further details are explained in the implementation. - * - * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, - * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. * * XXH3 implementation is portable: * it has a generic C90 formulation that can be compiled on any platform, - * all implementations generage exactly the same hash value on all platforms. + * all implementations generate exactly the same hash value on all platforms. * Starting from v0.8.0, it's also labelled "stable", meaning that * any future version will also generate the same hash value. * @@ -811,53 +1102,106 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src); * * The API supports one-shot hashing, streaming mode, and custom secrets. */ - /*-********************************************************************** - * XXH3 64-bit variant - ************************************************************************/ +* XXH3 64-bit variant +************************************************************************/ -/* XXH3_64bits(): - * default 64-bit variant, using default secret and default seed of 0. - * It's the fastest variant. */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *data, size_t len); +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); -/* - * XXH3_64bits_withSeed(): - * This variant generates a custom secret on the fly - * based on default secret altered using the `seed` value. +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * * While this operation is decently fast, note that it's not completely free. - * Note: seed==0 produces the same results as XXH3_64bits(). + * + * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void *data, size_t len, - XXH64_hash_t seed); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); - /*! - * The bare minimum size for a custom secret. - * - * @see - * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), - * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). - */ - #define XXH3_SECRET_SIZE_MIN 136 +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); -/* - * XXH3_64bits_withSecret(): - * It's possible to provide any blob of bytes as a "secret" to generate the - * hash. This makes it more difficult for an external actor to prepare an - * intentional collision. The main condition is that secretSize *must* be large - * enough (>= XXH3_SECRET_SIZE_MIN). However, the quality of produced hash - * values depends on secret's entropy. Technically, the secret must look like a - * bunch of random bytes. Avoid "trivial" or structured data such as repeated - * sequences or a text document. Whenever unsure about the "randomness" of the - * blob of bytes, consider relabelling it as a "custom seed" instead, and employ - * "XXH3_generateSecret()" (see below) to generate a high entropy secret derived - * from the custom seed. - */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *data, size_t len, - const void *secret, - size_t secretSize); /******* Streaming *******/ +#ifndef XXH_NO_STREAM /* * Streaming requires state maintenance. * This operation costs memory and CPU. @@ -866,52 +1210,143 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *data, size_t len, */ /*! - * @brief The state struct for the XXH3 streaming API. + * @brief The opaque state struct for the XXH3 streaming API. * * @see XXH3_state_s for details. + * @see @ref streaming_example "Streaming Example" */ -typedef struct XXH3_state_s XXH3_state_t; -XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr); -XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t *dst_state, - const XXH3_state_t *src_state); +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); -/* - * XXH3_64bits_reset(): - * Initialize with default parameters. - * digest will be equivalent to `XXH3_64bits()`. +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t *statePtr); -/* - * XXH3_64bits_reset_withSeed(): - * Generate a custom secret from `seed`, and store it into `statePtr`. - * digest will be equivalent to `XXH3_64bits_withSeed()`. +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + * @see @ref streaming_example "Streaming Example" + * */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t *statePtr, - XXH64_hash_t seed); -/* - * XXH3_64bits_reset_withSecret(): - * `secret` is referenced, it _must outlive_ the hash streaming session. - * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + * @see @ref streaming_example "Streaming Example" + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, * and the quality of produced hash values depends on secret's entropy * (secret's content should look like a bunch of random bytes). * When in doubt about the randomness of a candidate `secret`, * consider employing `XXH3_generateSecret()` instead (see below). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + * + * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret( - XXH3_state_t *statePtr, const void *secret, size_t secretSize); +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update(XXH3_state_t *statePtr, - const void *input, - size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *statePtr); +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ /* note : canonical representation of XXH3 is the same as XXH64 * since they both produce XXH64_hash_t values */ + /*-********************************************************************** - * XXH3 128-bit variant - ************************************************************************/ +* XXH3 128-bit variant +************************************************************************/ /*! * @brief The return value from 128-bit hashes. @@ -920,21 +1355,80 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *statePtr); * endianness. */ typedef struct { - - XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ - XXH64_hash_t high64; /*!< `value >> 64` */ - + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ } XXH128_hash_t; -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void *data, size_t len); -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void *data, size_t len, - XXH64_hash_t seed); -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *data, - size_t len, - const void *secret, - size_t secretSize); +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); /******* Streaming *******/ +#ifndef XXH_NO_STREAM /* * Streaming requires state maintenance. * This operation costs memory and CPU. @@ -944,73 +1438,193 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *data, * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). * Use already declared XXH3_createState() and XXH3_freeState(). * - * All reset and streaming functions have same meaning as their 64-bit - * counterpart. + * All reset and streaming functions have same meaning as their 64-bit counterpart. */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t *statePtr); -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t *statePtr, - XXH64_hash_t seed); -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret( - XXH3_state_t *statePtr, const void *secret, size_t secretSize); - -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t *statePtr, - const void *input, - size_t length); -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t *statePtr); - -/* Following helper functions make it possible to compare XXH128_hast_t values. - * Since XXH128_hash_t is a structure, this capability is not offered by the - * language. - * Note: For better performance, these functions can be inlined using - * XXH_INLINE_ALL */ - /*! - * XXH128_isEqual(): - * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + * + * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); /*! - * XXH128_cmp(): + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. * - * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. * - * return: >0 if *h128_1 > *h128_2 - * =0 if *h128_1 == *h128_2 - * <0 if *h128_1 < *h128_2 - */ -XXH_PUBLIC_API int XXH128_cmp(const void *h128_1, const void *h128_2); - + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * @brief Compares two @ref XXH128_hash_t + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); + + /******* Canonical representation *******/ -typedef struct { +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; - unsigned char digest[sizeof(XXH128_hash_t)]; -} XXH128_canonical_t; +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); -XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t *dst, - XXH128_hash_t hash); -XXH_PUBLIC_API XXH128_hash_t -XXH128_hashFromCanonical(const XXH128_canonical_t *src); +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); - #endif /* XXH_NO_LONG_LONG */ + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ /*! * @} */ -#endif /* XXHASH_H_5627135585666179 */ +#endif /* XXHASH_H_5627135585666179 */ + + #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) - #define XXHASH_H_STATIC_13879238742 +#define XXHASH_H_STATIC_13879238742 /* **************************************************************************** * This section contains declarations which are not guaranteed to remain stable. * They may change in future versions, becoming incompatible with a different * version of the library. * These declarations should only be used with static linking. * Never use them in association with dynamic linking! - ***************************************************************************** - */ + ***************************************************************************** */ /* * These definitions are only present to allow static allocation @@ -1031,23 +1645,16 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src); * @see XXH64_state_s, XXH3_state_s */ struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ - XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ - XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref - total_len_32 overflow) */ - XXH32_hash_t v1; /*!< First accumulator lane */ - XXH32_hash_t v2; /*!< Second accumulator lane */ - XXH32_hash_t v3; /*!< Third accumulator lane */ - XXH32_hash_t v4; /*!< Fourth accumulator lane */ - XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as - unsigned char[16]. */ - XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ - XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may - be removed. */ -}; /* typedef'd to XXH32_state_t */ - - #ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ /*! * @internal @@ -1062,64 +1669,57 @@ struct XXH32_state_s { * @see XXH32_state_s, XXH3_state_s */ struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif - XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ - XXH64_hash_t v1; /*!< First accumulator lane */ - XXH64_hash_t v2; /*!< Second accumulator lane */ - XXH64_hash_t v3; /*!< Third accumulator lane */ - XXH64_hash_t v4; /*!< Fourth accumulator lane */ - XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as - unsigned char[32]. */ - XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ - XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ - XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it - may be removed. */ - -}; /* typedef'd to XXH64_state_t */ - - #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 \ - */ - #include - #define XXH_ALIGN(n) alignas(n) - #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ - /* In C++ alignas() is a keyword */ - #define XXH_ALIGN(n) alignas(n) - #elif defined(__GNUC__) - #define XXH_ALIGN(n) __attribute__((aligned(n))) - #elif defined(_MSC_VER) - #define XXH_ALIGN(n) __declspec(align(n)) - #else - #define XXH_ALIGN(n) /* disabled */ - #endif - - /* Old GCC versions only accept the attribute after the type in structures. - */ - #if !(defined(__STDC_VERSION__) && \ - (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ - && !(defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ - && defined(__GNUC__) - #define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) - #else - #define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type - #endif - - /*! - * @brief The size of the internal XXH3 buffer. - * - * This is the optimal update size for incremental hashing. - * - * @see XXH3_64b_update(), XXH3_128b_update(). - */ - #define XXH3_INTERNALBUFFER_SIZE 256 +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif - /*! - * @brief Default size of the secret buffer (and @ref XXH3_kSecret). - * - * This is the size used in @ref XXH3_kSecret and the seeded functions. - * - * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. - */ - #define XXH3_SECRET_DEFAULT_SIZE 192 +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 /*! * @internal @@ -1144,111 +1744,284 @@ struct XXH64_state_s { * @see XXH32_state_s, XXH64_state_s */ struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER - XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); - /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref - * XXH64_state_s */ - XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); - /*!< Used to store a custom secret generated from a seed. */ - XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); - /*!< The internal buffer. @see XXH32_state_s::mem32 */ - XXH32_hash_t bufferedSize; - /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ - XXH32_hash_t reserved32; - /*!< Reserved field. Needed for padding on 64-bit. */ - size_t nbStripesSoFar; - /*!< Number or stripes processed. */ - XXH64_hash_t totalLen; - /*!< Total length hashed. 64-bit even on 32-bit targets. */ - size_t nbStripesPerBlock; - /*!< Number of stripes per block. */ - size_t secretLimit; - /*!< Size of @ref customSecret or @ref extSecret */ - XXH64_hash_t seed; - /*!< Seed for _withSeed variants. Must be zero otherwise, @see - * XXH3_INITSTATE() */ - XXH64_hash_t reserved64; - /*!< Reserved field. */ - const unsigned char *extSecret; - /*!< Reference to an external secret for the _withSecret variants, NULL - * for other variants. */ - /* note: there may be some padding at the end due to alignment on 64 bytes */ - -}; /* typedef'd to XXH3_state_t */ - - #undef XXH_ALIGN_MEMBER - - /*! - * @brief Initializes a stack-allocated `XXH3_state_s`. - * - * When the @ref XXH3_state_t structure is merely emplaced on stack, - * it should be initialized with XXH3_INITSTATE() or a memset() - * in case its first reset uses XXH3_NNbits_reset_withSeed(). - * This init can be omitted if the first reset uses default or _withSecret - * mode. This operation isn't necessary when the state is created with - * XXH3_createState(). Note that this doesn't prepare the state for a - * streaming operation, it's still necessary to use XXH3_NNbits_reset*() - * afterwards. - */ - #define XXH3_INITSTATE(XXH3_state_ptr) \ - { (XXH3_state_ptr)->seed = 0; } +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) -/* === Experimental API === */ -/* Symbols defined below must be considered tied to a specific library version. + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); -/* - * XXH3_generateSecret(): + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/*! + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. + * + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. * - * Derive a high-entropy secret from any user-defined content, named customSeed. - * The generated secret can be used in combination with `*_withSecret()` - * functions. The `_withSecret()` variants are useful to provide a higher level - * of protection than 64-bit seed, as it becomes much more difficult for an - * external actor to guess how to impact the calculation logic. + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. * * The function accepts as input a custom seed of any length and any content, - * and derives from it a high-entropy secret of length XXH3_SECRET_DEFAULT_SIZE - * into an already allocated buffer secretBuffer. - * The generated secret is _always_ XXH_SECRET_DEFAULT_SIZE bytes long. + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. * * The generated secret can then be used with any `*_withSecret()` variant. - * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, - * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() * are part of this list. They all accept a `secret` parameter - * which must be very long for implementation reasons (>= XXH3_SECRET_SIZE_MIN) + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) * _and_ feature very high entropy (consist of random-looking bytes). - * These conditions can be a high bar to meet, so - * this function can be used to generate a secret of proper quality. + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. * - * customSeed can be anything. It can have any size, even small ones, - * and its content can be anything, even stupidly "low entropy" source such as a - * bunch of zeroes. The resulting `secret` will nonetheless provide all expected - * qualities. + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. * - * Supplying NULL as the customSeed copies the default secret into - * `secretBuffer`. When customSeedSize > 0, supplying NULL as customSeed is - * undefined behavior. + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode */ -XXH_PUBLIC_API void XXH3_generateSecret(void *secretBuffer, - const void *customSeed, - size_t customSeedSize); +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); -/* simple short-cut to pre-selected XXH3_128bits variant */ -XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, - XXH64_hash_t seed); +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either + * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param input The block of data to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif - #endif /* XXH_NO_LONG_LONG */ - #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) - #define XXH_IMPLEMENTATION - #endif +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ -#endif /* defined(XXH_STATIC_LINKING_ONLY) && \ - !defined(XXHASH_H_STATIC_13879238742) */ /* ======================================================================== */ /* ======================================================================== */ /* ======================================================================== */ + /*-********************************************************************** * xxHash implementation *-********************************************************************** @@ -1271,424 +2044,477 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, * which can then be linked into the final binary. ************************************************************************/ -#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) || \ - defined(XXH_IMPLEMENTATION)) && \ - !defined(XXH_IMPLEM_13a8737387) - #define XXH_IMPLEM_13a8737387 - - /* ************************************* - * Tuning parameters - ***************************************/ - - /*! - * @defgroup tuning Tuning parameters - * @{ - - * - * Various macros to control xxHash's behavior. - */ - #ifdef XXH_DOXYGEN - /*! - * @brief Define this to disable 64-bit code. - * - * Useful if only using the @ref xxh32_family and you have a strict C90 - * compiler. - */ - #define XXH_NO_LONG_LONG - #undef XXH_NO_LONG_LONG /* don't actually */ - /*! - * @brief Controls how unaligned memory is accessed. - * - * By default, access to unaligned memory is controlled by `memcpy()`, which - * is safe and portable. - * - * Unfortunately, on some target/compiler combinations, the generated - * assembly is sub-optimal. - * - * The below switch allow selection of a different access method - * in the search for improved performance. - * - * @par Possible options: - * - * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` - * @par - * Use `memcpy()`. Safe and portable. Note that most modern compilers - * will eliminate the function call and treat it as an unaligned access. - * - * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` - * @par - * Depends on compiler extensions and is therefore not portable. - * This method is safe _if_ your compiler supports it, - * and *generally* as fast or faster than `memcpy`. - * - * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast - * @par - * Casts directly and dereferences. This method doesn't depend on the - * compiler, but it violates the C standard as it directly dereferences - * an unaligned pointer. It can generate buggy code on targets which do not - * support unaligned memory accesses, but in some circumstances, it's - * the only known way to get the most performance. - * - * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift - * @par - * Also portable. This can generate the best code on old compilers which - * don't inline small `memcpy()` calls, and it might also be faster on - * big-endian systems which lack a native byteswap instruction. However, - * some compilers will emit literal byteshifts even if the target supports - * unaligned access. - * . - * - * @warning - * Methods 1 and 2 rely on implementation-defined behavior. Use these with - * care, as what works on one compiler/platform/optimization level may - * cause another to read garbage data or even crash. - * - * See https://stackoverflow.com/a/32095106/646947 for details. - * - * Prefer these methods in priority order (0 > 3 > 1 > 2) - */ - #define XXH_FORCE_MEMORY_ACCESS 0 - /*! - * @def XXH_ACCEPT_NULL_INPUT_POINTER - * @brief Whether to add explicit `NULL` checks. - * - * If the input pointer is `NULL` and the length is non-zero, xxHash's - * default behavior is to dereference it, triggering a segfault. - * - * When this macro is enabled, xxHash actively checks the input for a null - * pointer. If it is, the result for null input pointers is the same as a - * zero-length input. - */ - #define XXH_ACCEPT_NULL_INPUT_POINTER 0 - /*! - * @def XXH_FORCE_ALIGN_CHECK - * @brief If defined to non-zero, adds a special path for aligned inputs - * (XXH32() and XXH64() only). - * - * This is an important performance trick for architectures without decent - * unaligned memory access performance. - * - * It checks for input alignment, and when conditions are met, uses a "fast - * path" employing direct 32-bit/64-bit reads, resulting in _dramatically - * faster_ read speed. - * - * The check costs one initial branch per hash, which is generally - * negligible, but not zero. - * - * Moreover, it's not useful to generate an additional code path if memory - * access uses the same instruction for both aligned and unaligned - * addresses (e.g. x86 and aarch64). - * - * In these cases, the alignment check can be removed by setting this macro - * to 0. Then the code will always use unaligned memory access. Align check - * is automatically disabled on x86, x64 & arm64, which are platforms known - * to offer good unaligned memory accesses performance. - * - * This option does not affect XXH3 (only XXH32 and XXH64). - */ - #define XXH_FORCE_ALIGN_CHECK 0 - - /*! - * @def XXH_NO_INLINE_HINTS - * @brief When non-zero, sets all functions to `static`. - * - * By default, xxHash tries to force the compiler to inline almost all - * internal functions. - * - * This can usually improve performance due to reduced jumping and improved - * constant folding, but significantly increases the size of the binary - * which might not be favorable. - * - * Additionally, sometimes the forced inlining can be detrimental to - * performance, depending on the architecture. - * - * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the - * compiler full control on whether to inline or not. - * - * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using - * -fno-inline with GCC or Clang, this will automatically be defined. - */ - #define XXH_NO_INLINE_HINTS 0 - - /*! - * @def XXH_REROLL - * @brief Whether to reroll `XXH32_finalize`. - * - * For performance, `XXH32_finalize` uses an unrolled loop - * in the form of a switch statement. - * - * This is not always desirable, as it generates larger code, - * and depending on the architecture, may even be slower - * - * This is automatically defined with `-Os`/`-Oz` on GCC and Clang. - */ - #define XXH_REROLL 0 +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 - /*! - * @internal - * @brief Redefines old internal names. - * - * For compatibility with code that uses xxHash's internals before the names - * were changed to improve namespacing. There is no other reason to use - * this. - */ - #define XXH_OLD_NAMES - #undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ - #endif /* XXH_DOXYGEN */ - /*! - * @} - */ - - #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command \ - line for example */ - /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ - #if !defined(__clang__) && \ - ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ - (defined(__GNUC__) && \ - ((defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ - (defined(__mips__) && (__mips <= 5 || __mips_isa_rev < 6) && \ - (!defined(__mips16) || defined(__mips_mips16e2)))))) - #define XXH_FORCE_MEMORY_ACCESS 1 - #endif - #endif - - #ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ - #define XXH_ACCEPT_NULL_INPUT_POINTER 0 - #endif - - #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ - #if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || \ - defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ - #define XXH_FORCE_ALIGN_CHECK 0 - #else - #define XXH_FORCE_ALIGN_CHECK 1 - #endif - #endif - - #ifndef XXH_NO_INLINE_HINTS - #if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ - || defined(__NO_INLINE__) /* -O0, -fno-inline */ - #define XXH_NO_INLINE_HINTS 1 - #else - #define XXH_NO_INLINE_HINTS 0 - #endif - #endif - - #ifndef XXH_REROLL - #if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ || \ - (defined(__GNUC__) && !defined(__clang__)) - /* The if/then loop is preferable to switch/case on gcc (on x64) */ - #define XXH_REROLL 1 - #else - #define XXH_REROLL 0 - #endif - #endif - - /*! - * @defgroup impl Implementation - * @{ - - */ - - /* ************************************* - * Includes & Memory related functions - ***************************************/ - /* - * Modify the local functions below should you wish to use - * different memory routines for malloc() and free() - */ - #include +/* ************************************* +* Tuning parameters +***************************************/ /*! - * @internal - * @brief Modify this function to use a different routine than malloc(). + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. */ -static void *XXH_malloc(size_t s) { - - return malloc(s); - -} - +#ifdef XXH_DOXYGEN /*! - * @internal - * @brief Modify this function to use a different routine than free(). + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. */ -static void XXH_free(void *p) { - - free(p); - -} - - #include - +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ /*! - * @internal - * @brief Modify this function to use a different routine than memcpy(). + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) */ -static void *XXH_memcpy(void *dest, const void *src, size_t size) { - - return memcpy(dest, src, size); - -} - - #include /* ULLONG_MAX */ - - /* ************************************* - * Compiler Specific Options - ***************************************/ - #ifdef _MSC_VER /* Visual Studio warning fix */ - #pragma warning(disable : 4127) /* disable: C4127: conditional expression \ - is constant */ - #endif - - #if XXH_NO_INLINE_HINTS /* disable inlining hints */ - #if defined(__GNUC__) - #define XXH_FORCE_INLINE static __attribute__((unused)) - #else - #define XXH_FORCE_INLINE static - #endif - #define XXH_NO_INLINE static - /* enable inlining hints */ - #elif defined(_MSC_VER) /* Visual Studio */ - #define XXH_FORCE_INLINE static __forceinline - #define XXH_NO_INLINE static __declspec(noinline) - #elif defined(__GNUC__) - #define XXH_FORCE_INLINE \ - static __inline__ __attribute__((always_inline, unused)) - #define XXH_NO_INLINE static __attribute__((noinline)) - #elif defined(__cplusplus) || \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ - #define XXH_FORCE_INLINE static inline - #define XXH_NO_INLINE static - #else - #define XXH_FORCE_INLINE static - #define XXH_NO_INLINE static - #endif - - /* ************************************* - * Debug - ***************************************/ - /*! - * @ingroup tuning - * @def XXH_DEBUGLEVEL - * @brief Sets the debugging level. - * - * XXH_DEBUGLEVEL is expected to be defined externally, typically via the - * compiler's command line options. The value must be a number. - */ - #ifndef XXH_DEBUGLEVEL - #ifdef DEBUGLEVEL /* backwards compat */ - #define XXH_DEBUGLEVEL DEBUGLEVEL - #else - #define XXH_DEBUGLEVEL 0 - #endif - #endif - - #if (XXH_DEBUGLEVEL >= 1) - #include /* note: can still be disabled with NDEBUG */ - #define XXH_ASSERT(c) assert(c) - #else - #define XXH_ASSERT(c) ((void)0) - #endif - - /* note: use after variable declarations */ - #ifndef XXH_STATIC_ASSERT - #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ - #include - #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ - do { \ - \ - static_assert((c), m); \ - \ - } while (0) - - #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ - #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ - do { \ - \ - static_assert((c), m); \ - \ - } while (0) - - #else - #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ - do { \ - \ - struct xxh_sa { \ - \ - char x[(c) ? 1 : -1]; \ - \ - }; \ - \ - } while (0) - - #endif - #define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c), #c) - #endif - - /*! - * @internal - * @def XXH_COMPILER_GUARD(var) - * @brief Used to prevent unwanted optimizations for @p var. - * - * It uses an empty GCC inline assembly statement with a register constraint - * which forces @p var into a general purpose register (eg eax, ebx, ecx - * on x86) and marks it as modified. - * - * This is used in a few places to avoid unwanted autovectorization (e.g. - * XXH32_round()). All vectorization we want is explicit via intrinsics, - * and _usually_ isn't wanted elsewhere. - * - * We also use it to prevent unwanted constant folding for AArch64 in - * XXH3_initCustomSecret_scalar(). - */ - #ifdef __GNUC__ - #define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r"(var)) - #else - #define XXH_COMPILER_GUARD(var) ((void)0) - #endif - - /* ************************************* - * Basic Types - ***************************************/ - #if !defined(__VMS) && \ - (defined(__cplusplus) || \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) - #include -typedef uint8_t xxh_u8; - #else -typedef unsigned char xxh_u8; - #endif -typedef XXH32_hash_t xxh_u32; +# define XXH_FORCE_MEMORY_ACCESS 0 - #ifdef XXH_OLD_NAMES - #define BYTE xxh_u8 - #define U8 xxh_u8 - #define U32 xxh_u32 - #endif - -/* *** Memory access *** */ +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 /*! - * @internal - * @fn xxh_u32 XXH_read32(const void* ptr) - * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). * - * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * This is an important performance trick for architectures without decent + * unaligned memory access performance. * - * @param ptr The pointer to read from. - * @return The 32-bit native endian integer from the bytes at @p ptr. + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). */ +# define XXH_FORCE_ALIGN_CHECK 0 /*! - * @internal - * @fn xxh_u32 XXH_readLE32(const void* ptr) - * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. * - * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. * - * @param ptr The pointer to read from. - * @return The 32-bit little endian integer from the bytes at @p ptr. + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#endif /* XXH_NO_STDLIB */ + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. */ /*! @@ -1719,288 +2545,304 @@ typedef XXH32_hash_t xxh_u32; * @return The 32-bit little endian integer from the bytes at @p ptr. */ - #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) - /* - * Manual byteshift. Best for old compilers which don't inline memcpy. - * We actually directly use XXH_readLE32 and XXH_readBE32. - */ - #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 2)) +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) /* * Force direct memory access. Only works on CPU which support unaligned memory * access in hardware. */ -static xxh_u32 XXH_read32(const void *memPtr) { +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } - return *(const xxh_u32 *)memPtr; +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); } - #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 1)) +#else - /* - * __pack instructions are safer but compiler specific, hence potentially - * problematic for some compilers. - * - * Currently only defined for GCC and ICC. - */ - #ifdef XXH_OLD_NAMES -typedef union { +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} - xxh_u32 u32; +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ -} __attribute__((packed)) unalign; - #endif -static xxh_u32 XXH_read32(const void *ptr) { +/* *** Endianness *** */ - typedef union { +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif - xxh_u32 u32; - } __attribute__((packed)) xxh_unalign; - return ((const xxh_unalign *)ptr)->u32; -} +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + - #else /* - * Portable and safe solution. Generally efficient. - * see: https://stackoverflow.com/a/32095106/646947 + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 */ -static xxh_u32 XXH_read32(const void *memPtr) { - xxh_u32 val; - memcpy(&val, memPtr, sizeof(val)); - return val; +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() -} +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) - #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ +#else +# define XXH_UNREACHABLE() +#endif - /* *** Endianness *** */ +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif - /*! - * @ingroup tuning - * @def XXH_CPU_LITTLE_ENDIAN - * @brief Whether the target is little endian. - * - * Defined to 1 if the target is little endian, or 0 if it is big endian. - * It can be defined externally, for example on the compiler command line. - * - * If it is not defined, - * a runtime check (which is usually constant folded) is used instead. - * - * @note - * This is not necessarily defined to an integer constant. - * - * @see XXH_isLittleEndian() for the runtime check. - */ - #ifndef XXH_CPU_LITTLE_ENDIAN - /* - * Try to detect endianness automatically, to avoid the nonstandard behavior - * in `XXH_isLittleEndian()` - */ - #if defined(_WIN32) /* Windows is always little endian */ \ - || defined(__LITTLE_ENDIAN__) || \ - (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) - #define XXH_CPU_LITTLE_ENDIAN 1 - #elif defined(__BIG_ENDIAN__) || \ - (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) - #define XXH_CPU_LITTLE_ENDIAN 0 - #else /*! * @internal - * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. * - * Most compilers will constant fold this. + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. */ -static int XXH_isLittleEndian(void) { - - /* - * Portable and well-defined behavior. - * Don't use static: it is detrimental to performance. - */ - const union { - - xxh_u32 u; - xxh_u8 c[4]; - - } one = {1}; - - return one.c[0]; - -} - - #define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() - #endif - #endif - - /* **************************************** - * Compiler-specific Functions and Macros - ******************************************/ - #define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - - #ifdef __has_builtin - #define XXH_HAS_BUILTIN(x) __has_builtin(x) - #else - #define XXH_HAS_BUILTIN(x) 0 - #endif - - /*! - * @internal - * @def XXH_rotl32(x,r) - * @brief 32-bit rotate left. - * - * @param x The 32-bit integer to be rotated. - * @param r The number of bits to rotate. - * @pre - * @p r > 0 && @p r < 32 - * @note - * @p x and @p r may be evaluated multiple times. - * @return The rotated result. - */ - #if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) && \ - XXH_HAS_BUILTIN(__builtin_rotateleft64) - #define XXH_rotl32 __builtin_rotateleft32 - #define XXH_rotl64 __builtin_rotateleft64 - /* Note: although _rotl exists for minGW (GCC under windows), performance - * seems poor */ - #elif defined(_MSC_VER) - #define XXH_rotl32(x, r) _rotl(x, r) - #define XXH_rotl64(x, r) _rotl64(x, r) - #else - #define XXH_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) - #define XXH_rotl64(x, r) (((x) << (r)) | ((x) >> (64 - (r)))) - #endif - - /*! - * @internal - * @fn xxh_u32 XXH_swap32(xxh_u32 x) - * @brief A 32-bit byteswap. - * - * @param x The 32-bit integer to byteswap. - * @return @p x, byteswapped. - */ - #if defined(_MSC_VER) /* Visual Studio */ - #define XXH_swap32 _byteswap_ulong - #elif XXH_GCC_VERSION >= 403 - #define XXH_swap32 __builtin_bswap32 - #else -static xxh_u32 XXH_swap32(xxh_u32 x) { - - return ((x << 24) & 0xff000000) | ((x << 8) & 0x00ff0000) | - ((x >> 8) & 0x0000ff00) | ((x >> 24) & 0x000000ff); - -} - - #endif +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + /* *************************** - * Memory reads - *****************************/ +* Memory reads +*****************************/ /*! * @internal * @brief Enum to indicate whether a pointer is aligned. */ typedef enum { - - XXH_aligned, /*!< Aligned */ - XXH_unaligned /*!< Possibly unaligned */ - + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ } XXH_alignment; - /* - * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. - * - * This is ideal for older compilers which don't inline memcpy. - */ - #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) - -XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void *memPtr) { - - const xxh_u8 *bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[0] | ((xxh_u32)bytePtr[1] << 8) | ((xxh_u32)bytePtr[2] << 16) | - ((xxh_u32)bytePtr[3] << 24); +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); } -XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void *memPtr) { - - const xxh_u8 *bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[3] | ((xxh_u32)bytePtr[2] << 8) | ((xxh_u32)bytePtr[1] << 16) | - ((xxh_u32)bytePtr[0] << 24); - +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); } - #else -XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void *ptr) { - - return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); } -static xxh_u32 XXH_readBE32(const void *ptr) { - - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); - +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); } +#endif - #endif - -XXH_FORCE_INLINE xxh_u32 XXH_readLE32_align(const void *ptr, - XXH_alignment align) { - - if (align == XXH_unaligned) { - - return XXH_readLE32(ptr); - - } else { - - return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32 *)ptr - : XXH_swap32(*(const xxh_u32 *)ptr); - - } - +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } } + /* ************************************* - * Misc - ***************************************/ +* Misc +***************************************/ /*! @ingroup public */ -XXH_PUBLIC_API unsigned XXH_versionNumber(void) { +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } - return XXH_VERSION_NUMBER; - -} /* ******************************************************************* - * 32-bit hash functions - *********************************************************************/ +* 32-bit hash functions +*********************************************************************/ /*! * @} - * @defgroup xxh32_impl XXH32 implementation + * @defgroup XXH32_impl XXH32 implementation * @ingroup impl + * + * Details on the XXH32 implementation. * @{ - */ -/* #define instead of static const, to be used as initializers */ - #define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ - #define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ - #define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ - #define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ - #define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ - - #ifdef XXH_OLD_NAMES - #define PRIME32_1 XXH_PRIME32_1 - #define PRIME32_2 XXH_PRIME32_2 - #define PRIME32_3 XXH_PRIME32_3 - #define PRIME32_4 XXH_PRIME32_4 - #define PRIME32_5 XXH_PRIME32_5 - #endif + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif /*! * @internal @@ -2013,50 +2855,51 @@ XXH_PUBLIC_API unsigned XXH_versionNumber(void) { * @param input The stripe of input to mix. * @return The mixed accumulator lane. */ -static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { - - acc += input * XXH_PRIME32_2; - acc = XXH_rotl32(acc, 13); - acc *= XXH_PRIME32_1; - #if (defined(__SSE4_1__) || defined(__aarch64__)) && \ - !defined(XXH_ENABLE_AUTOVECTORIZE) - /* - * UGLY HACK: - * A compiler fence is the only thing that prevents GCC and Clang from - * autovectorizing the XXH32 loop (pragmas and attributes don't work for some - * reason) without globally disabling SSE4.1. - * - * The reason we want to avoid vectorization is because despite working on - * 4 integers at a time, there are multiple factors slowing XXH32 down on - * SSE4: - * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on - * newer chips!) making it slightly slower to multiply four integers at - * once compared to four integers independently. Even when pmulld was - * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE - * just to multiply unless doing a long operation. - * - * - Four instructions are required to rotate, - * movqda tmp, v // not required with VEX encoding - * pslld tmp, 13 // tmp <<= 13 - * psrld v, 19 // x >>= 19 - * por v, tmp // x |= tmp - * compared to one for scalar: - * roll v, 13 // reliably fast across the board - * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason - * - * - Instruction level parallelism is actually more beneficial here because - * the SIMD actually serializes this operation: While v1 is rotating, v2 - * can load data, while v3 can multiply. SSE forces them to operate - * together. - * - * This is also enabled on AArch64, as Clang autovectorizes it incorrectly - * and it is pointless writing a NEON implementation that is basically the - * same speed as scalar for XXH32. - */ - XXH_COMPILER_GUARD(acc); - #endif - return acc; - +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; } /*! @@ -2066,38 +2909,20 @@ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { * The final mix ensures that all input bits have a chance to impact any bit in * the output digest, resulting in an unbiased distribution. * - * @param h32 The hash to avalanche. + * @param hash The hash to avalanche. * @return The avalanched hash. */ -static xxh_u32 XXH32_avalanche(xxh_u32 h32) { - - h32 ^= h32 >> 15; - h32 *= XXH_PRIME32_2; - h32 ^= h32 >> 13; - h32 *= XXH_PRIME32_3; - h32 ^= h32 >> 16; - return (h32); - +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; } - #define XXH_get32bits(p) XXH_readLE32_align(p, align) - - #define XXH_PROCESS1 \ - do { \ - \ - h32 += (*ptr++) * XXH_PRIME32_5; \ - h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ - \ - } while (0) - - #define XXH_PROCESS4 \ - do { \ - \ - h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ - ptr += 4; \ - h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ - \ - } while (0) +#define XXH_get32bits(p) XXH_readLE32_align(p, align) /*! * @internal @@ -2107,1515 +2932,1366 @@ static xxh_u32 XXH32_avalanche(xxh_u32 h32) { * This final stage will digest them to ensure that all input bytes are present * in the final mix. * - * @param h32 The hash to finalize. + * @param hash The hash to finalize. * @param ptr The pointer to the remaining input. * @param len The remaining length, modulo 16. * @param align Whether @p ptr is aligned. * @return The finalized hash. + * @see XXH64_finalize(). */ -static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, - XXH_alignment align) { - - /* Compact rerolled version */ - if (XXH_REROLL) { - - len &= 15; - while (len >= 4) { - - XXH_PROCESS4; - len -= 4; - - } - - while (len > 0) { - - XXH_PROCESS1; - --len; - - } - - return XXH32_avalanche(h32); - - } else { - - switch (len & 15) /* or switch(bEnd - p) */ { - - case 12: - XXH_PROCESS4; - XXH_FALLTHROUGH; - case 8: - XXH_PROCESS4; - XXH_FALLTHROUGH; - case 4: - XXH_PROCESS4; - return XXH32_avalanche(h32); - - case 13: - XXH_PROCESS4; - XXH_FALLTHROUGH; - case 9: - XXH_PROCESS4; - XXH_FALLTHROUGH; - case 5: - XXH_PROCESS4; - XXH_PROCESS1; - return XXH32_avalanche(h32); - - case 14: - XXH_PROCESS4; - XXH_FALLTHROUGH; - case 10: - XXH_PROCESS4; - XXH_FALLTHROUGH; - case 6: - XXH_PROCESS4; - XXH_PROCESS1; - XXH_PROCESS1; - return XXH32_avalanche(h32); - - case 15: - XXH_PROCESS4; - XXH_FALLTHROUGH; - case 11: - XXH_PROCESS4; - XXH_FALLTHROUGH; - case 7: - XXH_PROCESS4; - XXH_FALLTHROUGH; - case 3: - XXH_PROCESS1; - XXH_FALLTHROUGH; - case 2: - XXH_PROCESS1; - XXH_FALLTHROUGH; - case 1: - XXH_PROCESS1; - XXH_FALLTHROUGH; - case 0: - return XXH32_avalanche(h32); - +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ } - - XXH_ASSERT(0); - return h32; /* reaching this point is deemed impossible */ - - } - } - #ifdef XXH_OLD_NAMES - #define PROCESS1 XXH_PROCESS1 - #define PROCESS4 XXH_PROCESS4 - #else - #undef XXH_PROCESS1 - #undef XXH_PROCESS4 - #endif +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif /*! * @internal * @brief The implementation for @ref XXH32(). * - * @param input, len, seed Directly passed from @ref XXH32(). + * @param input , len , seed Directly passed from @ref XXH32(). * @param align Whether @p input is aligned. * @return The calculated hash. */ -XXH_FORCE_INLINE xxh_u32 XXH32_endian_align(const xxh_u8 *input, size_t len, - xxh_u32 seed, XXH_alignment align) { +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } - const xxh_u8 *bEnd = input ? input + len : NULL; - xxh_u32 h32; + h32 += (xxh_u32)len; - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) - if (input == NULL) { + return XXH32_finalize(h32, input, len&15, align); +} - len = 0; - bEnd = input = (const xxh_u8 *)(size_t)16; +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } - } + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} - #endif - if (len >= 16) { - - const xxh_u8 *const limit = bEnd - 15; - xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; - xxh_u32 v2 = seed + XXH_PRIME32_2; - xxh_u32 v3 = seed + 0; - xxh_u32 v4 = seed - XXH_PRIME32_1; - - do { - - v1 = XXH32_round(v1, XXH_get32bits(input)); - input += 4; - v2 = XXH32_round(v2, XXH_get32bits(input)); - input += 4; - v3 = XXH32_round(v3, XXH_get32bits(input)); - input += 4; - v4 = XXH32_round(v4, XXH_get32bits(input)); - input += 4; - - } while (input < limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + - XXH_rotl32(v4, 18); - - } else { - - h32 = seed + XXH_PRIME32_5; - - } - - h32 += (xxh_u32)len; - - return XXH32_finalize(h32, input, len & 15, align); - -} - -/*! @ingroup xxh32_family */ -XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, - XXH32_hash_t seed) { - - #if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, (const xxh_u8*)input, len); - return XXH32_digest(&state); - #else - if (XXH_FORCE_ALIGN_CHECK) { - - if ((((size_t)input) & 3) == - 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - return XXH32_endian_align((const xxh_u8 *)input, len, seed, XXH_aligned); - - } - - } - - return XXH32_endian_align((const xxh_u8 *)input, len, seed, XXH_unaligned); - #endif - -} /******* Hash streaming *******/ -/*! - * @ingroup xxh32_family - */ -XXH_PUBLIC_API XXH32_state_t *XXH32_createState(void) { - - return (XXH32_state_t *)XXH_malloc(sizeof(XXH32_state_t)); - -} - -/*! @ingroup xxh32_family */ -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr) { - - XXH_free(statePtr); - return XXH_OK; - -} - -/*! @ingroup xxh32_family */ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t *dstState, - const XXH32_state_t *srcState) { - - memcpy(dstState, srcState, sizeof(*dstState)); - +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; } -/*! @ingroup xxh32_family */ -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, - XXH32_hash_t seed) { - - XXH32_state_t state; /* using a local state to memcpy() in order to avoid - strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; - state.v2 = seed + XXH_PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - XXH_PRIME32_1; - /* do not write into reserved, planned to be removed in a future version */ - memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); - return XXH_OK; - +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); } -/*! @ingroup xxh32_family */ -XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, - const void *input, size_t len) { - - if (input == NULL) - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; return XXH_OK; - #else - return XXH_ERROR; - #endif - - { - - const xxh_u8 *p = (const xxh_u8 *)input; - const xxh_u8 *const bEnd = p + len; - - state->total_len_32 += (XXH32_hash_t)len; - state->large_len |= - (XXH32_hash_t)((len >= 16) | (state->total_len_32 >= 16)); - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((xxh_u8 *)(state->mem32) + state->memsize, input, len); - state->memsize += (XXH32_hash_t)len; - return XXH_OK; - - } - - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((xxh_u8 *)(state->mem32) + state->memsize, input, - 16 - state->memsize); - { - - const xxh_u32 *p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32)); - p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32)); - p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32)); - p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32)); - - } +} - p += 16 - state->memsize; - state->memsize = 0; +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; } - if (p <= bEnd - 16) { + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; - const xxh_u8 *const limit = bEnd - 16; - xxh_u32 v1 = state->v1; - xxh_u32 v2 = state->v2; - xxh_u32 v3 = state->v3; - xxh_u32 v4 = state->v4; + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); - do { - - v1 = XXH32_round(v1, XXH_readLE32(p)); - p += 4; - v2 = XXH32_round(v2, XXH_readLE32(p)); - p += 4; - v3 = XXH32_round(v3, XXH_readLE32(p)); - p += 4; - v4 = XXH32_round(v4, XXH_readLE32(p)); - p += 4; - - } while (p <= limit); + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } - } + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; - if (p < bEnd) { + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); - XXH_memcpy(state->mem32, p, (size_t)(bEnd - p)); - state->memsize = (unsigned)(bEnd - p); + } + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } } - } - - return XXH_OK; - + return XXH_OK; } -/*! @ingroup xxh32_family */ -XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *state) { - - xxh_u32 h32; - - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + - XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; - } else { - - h32 = state->v3 /* == seed */ + XXH_PRIME32_5; - - } - - h32 += state->total_len_32; + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } - return XXH32_finalize(h32, (const xxh_u8 *)state->mem32, state->memsize, - XXH_aligned); + h32 += state->total_len_32; + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); } +#endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ -/*! - * @ingroup xxh32_family - * The default return values from XXH functions are unsigned 32 and 64 bit - * integers. - * - * The canonical representation uses big endian convention, the same convention - * as human-readable numbers (large digits first). - * - * This way, hash values can be written into a file or buffer, remaining - * comparable across different systems. - * - * The following functions allow transformation of hash values to and from their - * canonical format. - */ -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, - XXH32_hash_t hash) { - - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - memcpy(dst, &hash, sizeof(*dst)); - +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); } - -/*! @ingroup xxh32_family */ -XXH_PUBLIC_API XXH32_hash_t -XXH32_hashFromCanonical(const XXH32_canonical_t *src) { - - return XXH_readBE32(src); - +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); } - #ifndef XXH_NO_LONG_LONG + +#ifndef XXH_NO_LONG_LONG /* ******************************************************************* - * 64-bit hash functions - *********************************************************************/ +* 64-bit hash functions +*********************************************************************/ /*! * @} * @ingroup impl * @{ - */ /******* Memory access *******/ typedef XXH64_hash_t xxh_u64; - #ifdef XXH_OLD_NAMES - #define U64 xxh_u64 - #endif - - #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) - /* - * Manual byteshift. Best for old compilers which don't inline memcpy. - * We actually directly use XXH_readLE64 and XXH_readBE64. - */ - #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory - * access in hardware */ -static xxh_u64 XXH_read64(const void *memPtr) { +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif - return *(const xxh_u64 *)memPtr; +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; } - #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 1)) - - /* - * __pack instructions are safer, but compiler specific, hence potentially - * problematic for some compilers. - * - * Currently only defined for GCC and ICC. - */ - #ifdef XXH_OLD_NAMES -typedef union { - - xxh_u32 u32; - xxh_u64 u64; - -} __attribute__((packed)) unalign64; - - #endif -static xxh_u64 XXH_read64(const void *ptr) { - - typedef union { - - xxh_u32 u32; - xxh_u64 u64; - - } __attribute__((packed)) xxh_unalign64; - - return ((const xxh_unalign64 *)ptr)->u64; +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); } - #else +#else /* * Portable and safe solution. Generally efficient. - * see: https://stackoverflow.com/a/32095106/646947 + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html */ -static xxh_u64 XXH_read64(const void *memPtr) { - - xxh_u64 val; - memcpy(&val, memPtr, sizeof(val)); - return val; - -} - - #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - - #if defined(_MSC_VER) /* Visual Studio */ - #define XXH_swap64 _byteswap_uint64 - #elif XXH_GCC_VERSION >= 403 - #define XXH_swap64 __builtin_bswap64 - #else -static xxh_u64 XXH_swap64(xxh_u64 x) { - - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); - +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); } +#endif - #endif - - /* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ - #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) - -XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void *memPtr) { - const xxh_u8 *bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[0] | ((xxh_u64)bytePtr[1] << 8) | ((xxh_u64)bytePtr[2] << 16) | - ((xxh_u64)bytePtr[3] << 24) | ((xxh_u64)bytePtr[4] << 32) | - ((xxh_u64)bytePtr[5] << 40) | ((xxh_u64)bytePtr[6] << 48) | - ((xxh_u64)bytePtr[7] << 56); +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); } -XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void *memPtr) { - - const xxh_u8 *bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[7] | ((xxh_u64)bytePtr[6] << 8) | ((xxh_u64)bytePtr[5] << 16) | - ((xxh_u64)bytePtr[4] << 24) | ((xxh_u64)bytePtr[3] << 32) | - ((xxh_u64)bytePtr[2] << 40) | ((xxh_u64)bytePtr[1] << 48) | - ((xxh_u64)bytePtr[0] << 56); - +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); } - #else -XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void *ptr) { - - return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); } -static xxh_u64 XXH_readBE64(const void *ptr) { - - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); - +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); } +#endif - #endif - -XXH_FORCE_INLINE xxh_u64 XXH_readLE64_align(const void *ptr, - XXH_alignment align) { - - if (align == XXH_unaligned) - return XXH_readLE64(ptr); - else - return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64 *)ptr - : XXH_swap64(*(const xxh_u64 *)ptr); - +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); } - /******* xxh64 *******/ - /*! - * @} - * @defgroup xxh64_impl XXH64 implementation - * @ingroup impl - * @{ - - */ - /* #define rather that static const, to be used as initializers */ - #define XXH_PRIME64_1 \ - 0x9E3779B185EBCA87ULL /*!< \ - 0b1001111000110111011110011011000110000101111010111100101010000111 \ - */ - #define XXH_PRIME64_2 \ - 0xC2B2AE3D27D4EB4FULL /*!< \ - 0b1100001010110010101011100011110100100111110101001110101101001111 \ - */ - #define XXH_PRIME64_3 \ - 0x165667B19E3779F9ULL /*!< \ - 0b0001011001010110011001111011000110011110001101110111100111111001 \ - */ - #define XXH_PRIME64_4 \ - 0x85EBCA77C2B2AE63ULL /*!< \ - 0b1000010111101011110010100111011111000010101100101010111001100011 \ - */ - #define XXH_PRIME64_5 \ - 0x27D4EB2F165667C5ULL /*!< \ - 0b0010011111010100111010110010111100010110010101100110011111000101 \ - */ - - #ifdef XXH_OLD_NAMES - #define PRIME64_1 XXH_PRIME64_1 - #define PRIME64_2 XXH_PRIME64_2 - #define PRIME64_3 XXH_PRIME64_3 - #define PRIME64_4 XXH_PRIME64_4 - #define PRIME64_5 XXH_PRIME64_5 - #endif -static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) { - - acc += input * XXH_PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= XXH_PRIME64_1; - return acc; +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; } -static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) { - - val = XXH64_round(0, val); - acc ^= val; - acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; - return acc; - +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; } -static xxh_u64 XXH64_avalanche(xxh_u64 h64) { - - h64 ^= h64 >> 33; - h64 *= XXH_PRIME64_2; - h64 ^= h64 >> 29; - h64 *= XXH_PRIME64_3; - h64 ^= h64 >> 32; - return h64; - +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; } - #define XXH_get64bits(p) XXH_readLE64_align(p, align) - -static xxh_u64 XXH64_finalize(xxh_u64 h64, const xxh_u8 *ptr, size_t len, - XXH_alignment align) { - - len &= 31; - while (len >= 8) { - - xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); - ptr += 8; - h64 ^= k1; - h64 = XXH_rotl64(h64, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; - len -= 8; - - } - - if (len >= 4) { - - h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; - ptr += 4; - h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; - len -= 4; - - } - - while (len > 0) { - h64 ^= (*ptr++) * XXH_PRIME64_5; - h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; - --len; - - } - - return XXH64_avalanche(h64); +#define XXH_get64bits(p) XXH_readLE64_align(p, align) +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); } - #ifdef XXH_OLD_NAMES - #define PROCESS1_64 XXH_PROCESS1_64 - #define PROCESS4_64 XXH_PROCESS4_64 - #define PROCESS8_64 XXH_PROCESS8_64 - #else - #undef XXH_PROCESS1_64 - #undef XXH_PROCESS4_64 - #undef XXH_PROCESS8_64 - #endif - -XXH_FORCE_INLINE xxh_u64 XXH64_endian_align(const xxh_u8 *input, size_t len, - xxh_u64 seed, XXH_alignment align) { - - const xxh_u8 *bEnd = input ? input + len : NULL; - xxh_u64 h64; - - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) - if (input == NULL) { - - len = 0; - bEnd = input = (const xxh_u8 *)(size_t)32; - - } - - #endif - - if (len >= 32) { - - const xxh_u8 *const limit = bEnd - 32; - xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; - xxh_u64 v2 = seed + XXH_PRIME64_2; - xxh_u64 v3 = seed + 0; - xxh_u64 v4 = seed - XXH_PRIME64_1; - - do { - - v1 = XXH64_round(v1, XXH_get64bits(input)); - input += 8; - v2 = XXH64_round(v2, XXH_get64bits(input)); - input += 8; - v3 = XXH64_round(v3, XXH_get64bits(input)); - input += 8; - v4 = XXH64_round(v4, XXH_get64bits(input)); - input += 8; - - } while (input <= limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + - XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - - } else { - - h64 = seed + XXH_PRIME64_5; +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif - } +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input= 2 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH64_state_t state; XXH64_reset(&state, seed); XXH64_update(&state, (const xxh_u8*)input, len); return XXH64_digest(&state); - #else - if (XXH_FORCE_ALIGN_CHECK) { - - if ((((size_t)input) & 7) == - 0) { /* Input is aligned, let's leverage the speed advantage */ - return XXH64_endian_align((const xxh_u8 *)input, len, seed, XXH_aligned); - - } - - } +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } - return XXH64_endian_align((const xxh_u8 *)input, len, seed, XXH_unaligned); - - #endif + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif } /******* Hash Streaming *******/ - -/*! @ingroup xxh64_family*/ -XXH_PUBLIC_API XXH64_state_t *XXH64_createState(void) { - - return (XXH64_state_t *)XXH_malloc(sizeof(XXH64_state_t)); - -} - -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr) { - - XXH_free(statePtr); - return XXH_OK; - -} - -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t *dstState, - const XXH64_state_t *srcState) { - - memcpy(dstState, srcState, sizeof(*dstState)); - +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; } -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t *statePtr, - XXH64_hash_t seed) { - - XXH64_state_t state; /* use a local state to memcpy() in order to avoid - strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; - state.v2 = seed + XXH_PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - XXH_PRIME64_1; - /* do not write into reserved64, might be removed in a future version */ - memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved64)); - return XXH_OK; - +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); } -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *state, - const void *input, size_t len) { - - if (input == NULL) - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; return XXH_OK; - #else - return XXH_ERROR; - #endif - - { - - const xxh_u8 *p = (const xxh_u8 *)input; - const xxh_u8 *const bEnd = p + len; - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((xxh_u8 *)state->mem64) + state->memsize, input, len); - state->memsize += (xxh_u32)len; - return XXH_OK; - - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((xxh_u8 *)state->mem64) + state->memsize, input, - 32 - state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64 + 0)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64 + 1)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64 + 2)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64 + 3)); - p += 32 - state->memsize; - state->memsize = 0; +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; } - if (p + 32 <= bEnd) { - - const xxh_u8 *const limit = bEnd - 32; - xxh_u64 v1 = state->v1; - xxh_u64 v2 = state->v2; - xxh_u64 v3 = state->v3; - xxh_u64 v4 = state->v4; + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; - do { + state->total_len += len; - v1 = XXH64_round(v1, XXH_readLE64(p)); - p += 8; - v2 = XXH64_round(v2, XXH_readLE64(p)); - p += 8; - v3 = XXH64_round(v3, XXH_readLE64(p)); - p += 8; - v4 = XXH64_round(v4, XXH_readLE64(p)); - p += 8; - - } while (p <= limit); + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } - } + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; - if (p < bEnd) { + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); - XXH_memcpy(state->mem64, p, (size_t)(bEnd - p)); - state->memsize = (unsigned)(bEnd - p); + } + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } } - } - - return XXH_OK; - + return XXH_OK; } -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t *state) { - - xxh_u64 h64; - - if (state->total_len >= 32) { - - xxh_u64 const v1 = state->v1; - xxh_u64 const v2 = state->v2; - xxh_u64 const v3 = state->v3; - xxh_u64 const v4 = state->v4; - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + - XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; - } else { - - h64 = state->v3 /*seed*/ + XXH_PRIME64_5; - - } - - h64 += (xxh_u64)state->total_len; + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } - return XXH64_finalize(h64, (const xxh_u8 *)state->mem64, - (size_t)state->total_len, XXH_aligned); + h64 += (xxh_u64) state->total_len; + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); } +#endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, - XXH64_hash_t hash) { - - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - memcpy(dst, &hash, sizeof(*dst)); - +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); } -/*! @ingroup xxh64_family */ -XXH_PUBLIC_API XXH64_hash_t -XXH64_hashFromCanonical(const XXH64_canonical_t *src) { - - return XXH_readBE64(src); - +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); } - #ifndef XXH_NO_XXH3 - - /* ********************************************************************* - * XXH3 - * New generation hash designed for speed on small keys and vectorization - ************************************************************************ */ - /*! - * @} - * @defgroup xxh3_impl XXH3 implementation - * @ingroup impl - * @{ - - */ +#ifndef XXH_NO_XXH3 - /* === Compiler specifics === */ - - #if ((defined(sun) || defined(__sun)) && \ - __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested \ - with GCC 5.5 */ - #define XXH_RESTRICT /* disable */ - #elif defined(__STDC_VERSION__) && \ - __STDC_VERSION__ >= 199901L /* >= C99 */ - #define XXH_RESTRICT restrict - #else - /* Note: it might be useful to define __restrict or __restrict__ for - * some C++ compilers */ - #define XXH_RESTRICT /* disable */ - #endif - - #if (defined(__GNUC__) && (__GNUC__ >= 3)) || \ - (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || \ - defined(__clang__) - #define XXH_likely(x) __builtin_expect(x, 1) - #define XXH_unlikely(x) __builtin_expect(x, 0) - #else - #define XXH_likely(x) (x) - #define XXH_unlikely(x) (x) - #endif - - #if defined(__GNUC__) - #if defined(__AVX2__) - #include - #elif defined(__SSE2__) - #include - #elif defined(__ARM_NEON__) || defined(__ARM_NEON) - #define inline __inline__ /* circumvent a clang bug */ - #include - #undef inline - #endif - #elif defined(_MSC_VER) - #include - #endif - - /* - * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while - * remaining a true 64-bit/128-bit hash function. - * - * This is done by prioritizing a subset of 64-bit operations that can be - * emulated without too many steps on the average 32-bit machine. - * - * For example, these two lines seem similar, and run equally fast on - * 64-bit: - * - * xxh_u64 x; - * x ^= (x >> 47); // good - * x ^= (x >> 13); // bad - * - * However, to a 32-bit machine, there is a major difference. - * - * x ^= (x >> 47) looks like this: - * - * x.lo ^= (x.hi >> (47 - 32)); - * - * while x ^= (x >> 13) looks like this: - * - * // note: funnel shifts are not usually cheap. - * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); - * x.hi ^= (x.hi >> 13); - * - * The first one is significantly faster than the second, simply because - * the shift is larger than 32. This means: - * - All the bits we need are in the upper 32 bits, so we can ignore the - * lower 32 bits in the shift. - * - The shift result will always fit in the lower 32 bits, and - * therefore, we can ignore the upper 32 bits in the xor. - * - * Thanks to this optimization, XXH3 only requires these features to be - * efficient: - * - * - Usable unaligned access - * - A 32-bit or 64-bit ALU - * - If 32-bit, a decent ADC instruction - * - A 32 or 64-bit multiply with a 64-bit result - * - For the 128-bit variant, a decent byteswap helps short inputs. - * - * The first two are already required by XXH32, and almost all 32-bit and - * 64-bit platforms which can run XXH32 can run XXH3 efficiently. - * - * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is - * one notable exception. - * - * First of all, Thumb-1 lacks support for the UMULL instruction which - * performs the important long multiply. This means numerous __aeabi_lmul - * calls. - * - * Second of all, the 8 functional registers are just not enough. - * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic - * need Lo registers, and this shuffling results in thousands more MOVs - * than A32. - * - * A32 and T32 don't have this limitation. They can access all 14 - * registers, do a 32->64 multiply with UMULL, and the flexible operand - * allowing free shifts is helpful, too. - * - * Therefore, we do a quick sanity check. - * - * If compiling Thumb-1 for a target which supports ARM instructions, we - * will emit a warning, as it is not a "sane" platform to compile for. - * - * Usually, if this happens, it is because of an accident and you probably - * need to specify -march, as you likely meant to compile for a newer - * architecture. - * - * Credit: large sections of the vectorial and asm source code paths - * have been contributed by @easyaspi314 - */ - #if defined(__thumb__) && !defined(__thumb2__) && \ - defined(__ARM_ARCH_ISA_ARM) - #warning "XXH3 is highly inefficient without ARM or Thumb-2." - #endif - - /* ========================================== - * Vectorization detection - * ========================================== */ - - #ifdef XXH_DOXYGEN - /*! - * @ingroup tuning - * @brief Overrides the vectorization implementation chosen for XXH3. - * - * Can be defined to 0 to disable SIMD or any of the values mentioned in - * @ref XXH_VECTOR_TYPE. - * - * If this is not defined, it uses predefined macros to determine the - * best implementation. - */ - #define XXH_VECTOR XXH_SCALAR +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ /*! - * @ingroup tuning - * @brief Possible values for @ref XXH_VECTOR. - * - * Note that these are actually implemented as macros. - * - * If this is not defined, it is detected automatically. - * @ref XXH_X86DISPATCH overrides this. + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ */ -enum XXH_VECTOR_TYPE /* fake enum */ { - XXH_SCALAR = 0, /*!< Portable scalar version */ - XXH_SSE2 = 1, /*!< - * SSE2 for Pentium 4, Opteron, all x86_64. - * - * @note SSE2 is also guaranteed on Windows 10, macOS, and - * Android x86. - */ - XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ - XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ - XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ - XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +/* === Compiler specifics === */ -}; +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif - /*! - * @ingroup tuning - * @brief Selects the minimum alignment for XXH3's accumulators. - * - * When using SIMD, this should match the alignment reqired for said - * vector type, so, for example, 32 for AVX2. - * - * Default: Auto detected. - */ - #define XXH_ACC_ALIGN 8 - #endif - - /* Actual definition */ - #ifndef XXH_DOXYGEN - #define XXH_SCALAR 0 - #define XXH_SSE2 1 - #define XXH_AVX2 2 - #define XXH_AVX512 3 - #define XXH_NEON 4 - #define XXH_VSX 5 - #endif - - #ifndef XXH_VECTOR /* can be defined on command line */ - #if defined(__AVX512F__) - #define XXH_VECTOR XXH_AVX512 - #elif defined(__AVX2__) - #define XXH_VECTOR XXH_AVX2 - #elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) - #define XXH_VECTOR XXH_SSE2 - #elif defined(__GNUC__) /* msvc support maybe later */ \ - && (defined(__ARM_NEON__) || defined(__ARM_NEON)) && \ - (defined( \ - __LITTLE_ENDIAN__) /* We only support little endian NEON */ \ - || (defined(__BYTE_ORDER__) && \ - __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) - #define XXH_VECTOR XXH_NEON - #elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) || \ - (defined(__s390x__) && defined(__VEC__)) && \ - defined(__GNUC__) /* TODO: IBM XL */ - #define XXH_VECTOR XXH_VSX - #else - #define XXH_VECTOR XXH_SCALAR - #endif - #endif - - /* - * Controls the alignment of the accumulator, - * for compatibility with aligned vector loads, which are usually faster. - */ - #ifndef XXH_ACC_ALIGN - #if defined(XXH_X86DISPATCH) - #define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ - #elif XXH_VECTOR == XXH_SCALAR /* scalar */ - #define XXH_ACC_ALIGN 8 - #elif XXH_VECTOR == XXH_SSE2 /* sse2 */ - #define XXH_ACC_ALIGN 16 - #elif XXH_VECTOR == XXH_AVX2 /* avx2 */ - #define XXH_ACC_ALIGN 32 - #elif XXH_VECTOR == XXH_NEON /* neon */ - #define XXH_ACC_ALIGN 16 - #elif XXH_VECTOR == XXH_VSX /* vsx */ - #define XXH_ACC_ALIGN 16 - #elif XXH_VECTOR == XXH_AVX512 /* avx512 */ - #define XXH_ACC_ALIGN 64 - #endif - #endif - - #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 || \ - XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 - #define XXH_SEC_ALIGN XXH_ACC_ALIGN - #else - #define XXH_SEC_ALIGN 8 - #endif - - /* - * UGLY HACK: - * GCC usually generates the best code with -O3 for xxHash. - * - * However, when targeting AVX2, it is overzealous in its unrolling - * resulting in code roughly 3/4 the speed of Clang. - * - * There are other issues, such as GCC splitting _mm256_loadu_si256 into - * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization - * which only applies to Sandy and Ivy Bridge... which don't even support - * AVX2. - * - * That is why when compiling the AVX2 version, it is recommended to use - * either -O2 -mavx2 -march=haswell or -O2 -mavx2 - * -mno-avx256-split-unaligned-load for decent performance, or to use - * Clang instead. - * - * Fortunately, we can control the first one with a pragma that forces GCC - * into -O2, but the other one we can't control without "failed to inline - * always inline function due to target mismatch" warnings. - */ - #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ - && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__OPTIMIZE__) && \ - !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ - #pragma GCC push_options - #pragma GCC optimize("-O2") - #endif - - #if XXH_VECTOR == XXH_NEON - /* - * NEON's setup for vmlal_u32 is a little more complicated than it is on - * SSE2, AVX2, and VSX. - * - * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an - * upcast. - * - * To do the same operation, the 128-bit 'Q' register needs to be split - * into two 64-bit 'D' registers, performing this operation:: - * - * [ a | b ] | - * '---------. .--------' | | x | - * | .---------' '--------. | - * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 - * ] - * - * Due to significant changes in aarch64, the fastest method for aarch64 - * is completely different than the fastest method for ARMv7-A. - * - * ARMv7-A treats D registers as unions overlaying Q registers, so - * modifying D11 will modify the high half of Q5. This is similar to how - * modifying AH will only affect bits 8-15 of AX on x86. - * - * VZIP takes two registers, and puts even lanes in one register and odd - * lanes in the other. - * - * On ARMv7-A, this strangely modifies both parameters in place instead - * of taking the usual 3-operand form. - * - * Therefore, if we want to do this, we can simply use a D-form VZIP.32 - * on the lower and upper halves of the Q register to end up with the - * high and low halves where we want - all in one instruction. - * - * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif - * d10[1], d11[1] } - * - * Unfortunately we need inline assembly for this: Instructions - * modifying two registers at once is not possible in GCC or Clang's IR, - * and they have to create a copy. - * - * aarch64 requires a different approach. - * - * In order to make it easier to write a decent compiler for aarch64, - * many quirks were removed, such as conditional execution. - * - * NEON was also affected by this. - * - * aarch64 cannot access the high bits of a Q-form register, and writes - * to a D-form register zero the high bits, similar to how writes to - * W-form scalar registers (or DWORD registers on x86_64) work. - * - * The formerly free vget_high intrinsics now require a vext (with a few - * exceptions) - * - * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the - * equivalent of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to - * only modify one operand. - * - * The equivalent of the VZIP.32 on the lower and upper halves would be - * this mess: - * - * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], - * v0[1] } zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } zip2 - * v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } - * - * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 - * (SHRN): - * - * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); - * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); - * - * This is available on ARMv7-A, but is less efficient than a single - * VZIP.32. - */ +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif - /*! - * Function-like macro: - * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t - * &outHi) - * { +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif - * outLo = (uint32x2_t)(in & 0xFFFFFFFF); - * outHi = (uint32x2_t)(in >> 32); - * in = UNDEFINED; - * } - */ - #if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ - && defined(__GNUC__) && !defined(__aarch64__) && \ - !defined(__arm64__) - #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - \ - /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, \ - * %f0 = upper D half */ \ - /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 \ - */ \ - /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 \ - */ \ - __asm__("vzip.32 %e0, %f0" : "+w"(in)); \ - (outLo) = vget_low_u32(vreinterpretq_u32_u64(in)); \ - (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ - \ - } while (0) - - #else - #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - \ - (outLo) = vmovn_u64(in); \ - (outHi) = vshrn_n_u64((in), 32); \ - \ - } while (0) - - #endif - #endif /* XXH_VECTOR == XXH_NEON */ - - /* - * VSX and Z Vector helpers. - * - * This is very messy, and any pull requests to clean this up are welcome. - * - * There are a lot of problems with supporting VSX and s390x, due to - * inconsistent intrinsics, spotty coverage, and multiple endiannesses. - */ - #if XXH_VECTOR == XXH_VSX - #if defined(__s390x__) - #include - #else - /* gcc's altivec.h can have the unwanted consequence to - * unconditionally #define bool, vector, and pixel keywords, with bad - * consequences for programs already using these keywords for other - * purposes. The paragraph defining these macros is skipped when - * __APPLE_ALTIVEC__ is defined. - * __APPLE_ALTIVEC__ is _generally_ defined automatically by the - * compiler, but it seems that, in some cases, it isn't. Force the - * build macro to be defined, so that keywords are not altered. - */ - #if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) - #define __APPLE_ALTIVEC__ - #endif - #include - #endif +#if defined(_MSC_VER) +# include +#endif -typedef __vector unsigned long long xxh_u64x2; -typedef __vector unsigned char xxh_u8x16; -typedef __vector unsigned xxh_u32x4; - - #ifndef XXH_VSX_BE - #if defined(__BIG_ENDIAN__) || \ - (defined(__BYTE_ORDER__) && \ - __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) - #define XXH_VSX_BE 1 - #elif defined(__VEC_ELEMENT_REG_ORDER__) && \ - __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ - #warning \ - "-maltivec=be is not recommended. Please use native endianness." - #define XXH_VSX_BE 1 - #else - #define XXH_VSX_BE 0 - #endif - #endif /* !defined(XXH_VSX_BE) */ - - #if XXH_VSX_BE - #if defined(__POWER9_VECTOR__) || \ - (defined(__clang__) && defined(__s390x__)) - #define XXH_vec_revb vec_revb - #else +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN /*! - * A polyfill for POWER9's vec_revb(). + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. */ -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) { +# define XXH_ACC_ALIGN 8 +#endif - xxh_u8x16 const vByteSwap = {0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, - 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08}; - return vec_perm(val, val, vByteSwap); +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif -} +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif - #endif - #endif /* XXH_VSX_BE */ +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif -/*! - * Performs an unaligned vector load and byte swaps it on big endian. +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. */ -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) { +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# endif +#endif - xxh_u64x2 ret; - memcpy(&ret, ptr, sizeof(xxh_u64x2)); - #if XXH_VSX_BE - ret = XXH_vec_revb(ret); - #endif - return ret; +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif -} +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif - /* - * vec_mulo and vec_mule are very problematic intrinsics on PowerPC - * - * These intrinsics weren't added until GCC 8, despite existing for a - * while, and they are endian dependent. Also, their meaning swap - * depending on version. - * */ - #if defined(__s390x__) - /* s390x is always big endian, no issue on this platform */ - #define XXH_vec_mulo vec_mulo - #define XXH_vec_mule vec_mule - #elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) - /* Clang has a better way to control this, we can just use the builtin - * which doesn't swap. */ - #define XXH_vec_mulo __builtin_altivec_vmulouw - #define XXH_vec_mule __builtin_altivec_vmuleuw - #else -/* gcc needs inline assembly */ -/* Adapted from - * https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) { +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; - xxh_u64x2 result; - __asm__("vmulouw %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); - return result; +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); } +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include +# else +# include +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) { +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; - xxh_u64x2 result; - __asm__("vmuleuw %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); - return result; +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; } - #endif /* XXH_vec_mulo, XXH_vec_mule */ - #endif /* XXH_VECTOR == XXH_VSX */ +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ - /* prefetch - * can be disabled, by declaring XXH_NO_PREFETCH build macro */ - #if defined(XXH_NO_PREFETCH) - #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ - #else - #if defined(_MSC_VER) && \ - (defined(_M_X64) || \ - defined( \ - _M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ - #include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ - #define XXH_PREFETCH(ptr) \ - _mm_prefetch((const char *)(ptr), _MM_HINT_T0) - #elif defined(__GNUC__) && \ - ((__GNUC__ >= 4) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))) - #define XXH_PREFETCH(ptr) \ - __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) - #else - #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ - #endif - #endif /* XXH_NO_PREFETCH */ - /* ========================================== - * XXH3 default settings - * ========================================== */ +/* ========================================== + * XXH3 default settings + * ========================================== */ - #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ - #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) - #error "default keyset is not large enough" - #endif +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif /*! Pseudorandom secret taken directly from FARSH. */ -XXH_ALIGN(64) -static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { - - 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, - 0xf7, 0x21, 0xad, 0x1c, 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, - 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, 0xcb, 0x79, 0xe6, 0x4e, - 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, - 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, - 0x81, 0x3a, 0x26, 0x4c, 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, - 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, 0x71, 0x64, 0x48, 0x97, - 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, - 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, - 0xc7, 0x0b, 0x4f, 0x1d, 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, - 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, 0xea, 0xc5, 0xac, 0x83, - 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, - 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, - 0x29, 0xd4, 0x68, 0x9e, 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, - 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, 0x45, 0xcb, 0x3a, 0x8f, - 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, - +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, }; - #ifdef XXH_OLD_NAMES - #define kSecret XXH3_kSecret - #endif +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif - #ifdef XXH_DOXYGEN +#ifdef XXH_DOXYGEN /*! * @brief Calculates a 32-bit to 64-bit long multiply. * * Implemented as a macro. * - * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it - * doesn't need to (but it shouldn't need to anyways, it is about 7 instructions - * to do a 64x64 multiply...). Since we know that this will _always_ emit - * `MULL`, we use that instead of the normal method. + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. * - * If you are compiling for platforms like Thumb-1 and don't have a better - * option, you may also want to write your own long multiply routine here. + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. * * @param x, y Numbers to be multiplied * @return 64-bit product of the low 32 bits of @p x and @p y. */ -XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { - - return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); - -} - - #elif defined(_MSC_VER) && defined(_M_IX86) - #include - #define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) - #else - /* - * Downcast + upcast is usually better than masking on older compilers - * like GCC 4.2 (especially 32-bit ones), all without affecting newer - * compilers. - * - * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both - * operands and perform a full 64x64 multiply -- entirely redundant on - * 32-bit. - */ - #define XXH_mult32to64(x, y) \ - ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) - #endif +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif /*! * @brief Calculates a 64->128-bit long multiply. @@ -3623,157 +4299,167 @@ XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar * version. * - * @param lhs, rhs The 64-bit integers to be multiplied + * @param lhs , rhs The 64-bit integers to be multiplied * @return The 128-bit result represented in an @ref XXH128_hash_t. */ -static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { - - /* - * GCC/Clang __uint128_t method. - * - * On most 64-bit targets, GCC and Clang define a __uint128_t type. - * This is usually the best way as it usually uses a native long 64-bit - * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. - * - * Usually. - * - * Despite being a 32-bit platform, Clang (and emscripten) define this - * type despite not having the arithmetic for it. This results in a laggy - * compiler builtin call which calculates a full 128-bit multiply. - * In that case it is best to use the portable one. - * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 - */ - #if defined(__GNUC__) && !defined(__wasm__) && \ - defined(__SIZEOF_INT128__) || \ - (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) - - __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; - XXH128_hash_t r128; - r128.low64 = (xxh_u64)(product); - r128.high64 = (xxh_u64)(product >> 64); - return r128; +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) - /* - * MSVC for x64's _umul128 method. - * - * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 - * *HighProduct); - * - * This compiles to single operand MUL on x64. - */ - #elif defined(_M_X64) || defined(_M_IA64) - - #ifndef _MSC_VER - #pragma intrinsic(_umul128) - #endif - xxh_u64 product_high; - xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); - XXH128_hash_t r128; - r128.low64 = product_low; - r128.high64 = product_high; - return r128; - - #else - /* - * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. - * - * This is a fast and simple grade school multiply, which is shown below - * with base 10 arithmetic instead of base 0x100000000. - * - * 9 3 // D2 lhs = 93 - * x 7 5 // D2 rhs = 75 - * ---------- - * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 - * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 - * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 - * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 - * --------- - * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 - * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 - * --------- - * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 - * - * The reasons for adding the products like this are: - * 1. It avoids manual carry tracking. Just like how - * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. - * This avoids a lot of complexity. - * - * 2. It hints for, and on Clang, compiles to, the powerful UMAAL - * instruction available in ARM's Digital Signal Processing extension - * in 32-bit ARMv6 and later, which is shown below: - * - * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) - * { - - * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; - * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); - * *RdHi = (xxh_u32)(product >> 32); - * } - * - * This instruction was designed for efficient long multiplication, and - * allows this to be calculated in only 4 instructions at speeds - * comparable to some 64-bit ALUs. - * - * 3. It isn't terrible on other platforms. Usually this will be a couple - * of 32-bit ADD/ADCs. - */ - - /* First calculate all of the cross products. */ - xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); - xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); - xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); - xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); - - /* Now add the products together. These will never overflow. */ - xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; - xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; - xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); - - XXH128_hash_t r128; - r128.low64 = lower; - r128.high64 = upper; - return r128; - #endif + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif } /*! * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. * * The reason for the separate function is to prevent passing too many structs - * around by value. This will hopefully inline the multiply, but we don't force - * it. + * around by value. This will hopefully inline the multiply, but we don't force it. * - * @param lhs, rhs The 64-bit integers to multiply + * @param lhs , rhs The 64-bit integers to multiply * @return The low 64 bits of the product XOR'd by the high 64 bits. * @see XXH_mult64to128() */ -static xxh_u64 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { - - XXH128_hash_t product = XXH_mult64to128(lhs, rhs); - return product.low64 ^ product.high64; - +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; } /*! Seems to produce slightly better code on GCC for some reason. */ -XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) { - - XXH_ASSERT(0 <= shift && shift < 64); - return v64 ^ (v64 >> shift); - +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); } /* * This is a fast avalanche stage, * suitable when input bits are already partially mixed */ -static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) { - - h64 = XXH_xorshift64(h64, 37); - h64 *= 0x165667919E3779F9ULL; - h64 = XXH_xorshift64(h64, 32); - return h64; - +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; } /* @@ -3781,17 +4467,17 @@ static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) { * inspired by Pelle Evensen's rrmxmx * preferable when input has not been previously mixed */ -static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) { - - /* this mix is inspired by Pelle Evensen's rrmxmx */ - h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); - h64 *= 0x9FB21C651E98DF25ULL; - h64 ^= (h64 >> 35) + len; - h64 *= 0x9FB21C651E98DF25ULL; - return XXH_xorshift64(h64, 28); - +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); } + /* ========================================== * Short keys * ========================================== @@ -3800,8 +4486,7 @@ static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) { * favored lengths that were a multiple of 4 or 8. * * Instead of iterating over individual inputs, we use a set of single shot - * functions which piece together a range of lengths and operate in constant - * time. + * functions which piece together a range of lengths and operate in constant time. * * Additionally, the number of multiplies has been significantly reduced. This * reduces latency, especially when emulating 64-bit multiplies on 32-bit. @@ -3826,98 +4511,70 @@ static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) { * * This adds an extra layer of strength for custom secrets. */ -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_1to3_64b(const xxh_u8 *input, size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { - - XXH_ASSERT(input != NULL); - XXH_ASSERT(1 <= len && len <= 3); - XXH_ASSERT(secret != NULL); - /* - * len = 1: combined = { input[0], 0x01, input[0], input[0] } - * len = 2: combined = { input[1], 0x02, input[0], input[1] } - * len = 3: combined = { input[2], 0x03, input[0], input[1] } - */ - { - - xxh_u8 const c1 = input[0]; - xxh_u8 const c2 = input[len >> 1]; - xxh_u8 const c3 = input[len - 1]; - xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) | - ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); - xxh_u64 const bitflip = - (XXH_readLE32(secret) ^ XXH_readLE32(secret + 4)) + seed; - xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; - return XXH64_avalanche(keyed); - - } - +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } } -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_4to8_64b(const xxh_u8 *input, size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { - - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(4 <= len && len <= 8); - seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; - { - - xxh_u32 const input1 = XXH_readLE32(input); - xxh_u32 const input2 = XXH_readLE32(input + len - 4); - xxh_u64 const bitflip = - (XXH_readLE64(secret + 8) ^ XXH_readLE64(secret + 16)) - seed; - xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); - xxh_u64 const keyed = input64 ^ bitflip; - return XXH3_rrmxmx(keyed, len); - - } - +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } } -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_9to16_64b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { - - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(9 <= len && len <= 16); - { - - xxh_u64 const bitflip1 = - (XXH_readLE64(secret + 24) ^ XXH_readLE64(secret + 32)) + seed; - xxh_u64 const bitflip2 = - (XXH_readLE64(secret + 40) ^ XXH_readLE64(secret + 48)) - seed; - xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; - xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; - xxh_u64 const acc = len + XXH_swap64(input_lo) + input_hi + - XXH3_mul128_fold64(input_lo, input_hi); - return XXH3_avalanche(acc); - - } - +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } } -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_0to16_64b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { - - XXH_ASSERT(len <= 16); - { - - if (XXH_likely(len > 8)) - return XXH3_len_9to16_64b(input, len, secret, seed); - if (XXH_likely(len >= 4)) - return XXH3_len_4to8_64b(input, len, secret, seed); - if (len) return XXH3_len_1to3_64b(input, len, secret, seed); - return XXH64_avalanche( - seed ^ (XXH_readLE64(secret + 56) ^ XXH_readLE64(secret + 64))); - - } - +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } } /* @@ -3946,113 +4603,106 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_0to16_64b(const xxh_u8 *input, * by this, although it is always a good idea to use a proper seed if you care * about strength. */ -XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8 *XXH_RESTRICT input, - const xxh_u8 *XXH_RESTRICT secret, - xxh_u64 seed64) { - - #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like \ - XXH32 hack */ - /* - * UGLY HACK: - * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in - * slower code. - * - * By forcing seed64 into a register, we disrupt the cost model and - * cause it to scalarize. See `XXH32_round()` - * - * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, - * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on - * GCC 9.2, despite both emitting scalar code. - * - * GCC generates much better scalar code than Clang for the rest of XXH3, - * which is why finding a more optimal codepath is an interest. - */ - XXH_COMPILER_GUARD(seed64); - #endif - { - - xxh_u64 const input_lo = XXH_readLE64(input); - xxh_u64 const input_hi = XXH_readLE64(input + 8); - return XXH3_mul128_fold64(input_lo ^ (XXH_readLE64(secret) + seed64), - input_hi ^ (XXH_readLE64(secret + 8) - seed64)); - - } - +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } } /* For mid range keys, XXH3 uses a Mum-hash variant. */ -XXH_FORCE_INLINE XXH64_hash_t XXH3_len_17to128_64b( - const xxh_u8 *XXH_RESTRICT input, size_t len, - const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { - - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - (void)secretSize; - XXH_ASSERT(16 < len && len <= 128); - - { - - xxh_u64 acc = len * XXH_PRIME64_1; - if (len > 32) { - - if (len > 64) { - - if (len > 96) { - - acc += XXH3_mix16B(input + 48, secret + 96, seed); - acc += XXH3_mix16B(input + len - 64, secret + 112, seed); - +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); } - - acc += XXH3_mix16B(input + 32, secret + 64, seed); - acc += XXH3_mix16B(input + len - 48, secret + 80, seed); - - } - - acc += XXH3_mix16B(input + 16, secret + 32, seed); - acc += XXH3_mix16B(input + len - 32, secret + 48, seed); - + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); } - - acc += XXH3_mix16B(input + 0, secret + 0, seed); - acc += XXH3_mix16B(input + len - 16, secret + 16, seed); - - return XXH3_avalanche(acc); - - } - } - #define XXH3_MIDSIZE_MAX 240 - -XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( - const xxh_u8 *XXH_RESTRICT input, size_t len, - const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { - - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - (void)secretSize; - XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); - - #define XXH3_MIDSIZE_STARTOFFSET 3 - #define XXH3_MIDSIZE_LASTOFFSET 17 - - { - - xxh_u64 acc = len * XXH_PRIME64_1; - int const nbRounds = (int)len / 16; - int i; - for (i = 0; i < 8; i++) { - - acc += XXH3_mix16B(input + (16 * i), secret + (16 * i), seed); - - } - - acc = XXH3_avalanche(acc); - XXH_ASSERT(nbRounds >= 8); - #if defined(__clang__) /* Clang */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ +/*! + * @brief Maximum size of "short" key in bytes. + */ +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ /* * UGLY HACK: * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. @@ -4065,2205 +4715,2334 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( * converts them to the nonexistent "vmulq_u64" intrinsic, which is then * scalarized into an ugly mess of VMOV.32 instructions. * - * This mess is difficult to avoid without turning autovectorization - * off completely, but they are usually relatively minor and/or not - * worth it to fix. - * - * This loop is the easiest to fix, as unlike XXH32, this pragma - * _actually works_ because it is a loop vectorization instead of an - * SLP vectorization. - */ - #pragma clang loop vectorize(disable) - #endif - for (i = 8; i < nbRounds; i++) { - - acc += - XXH3_mix16B(input + (16 * i), - secret + (16 * (i - 8)) + XXH3_MIDSIZE_STARTOFFSET, seed); - - } - - /* last bytes */ - acc += XXH3_mix16B(input + len - 16, - secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, - seed); - return XXH3_avalanche(acc); - - } - -} - - /* ======= Long Keys ======= */ - - #define XXH_STRIPE_LEN 64 - #define XXH_SECRET_CONSUME_RATE \ - 8 /* nb of secret bytes consumed at each accumulation */ - #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) - - #ifdef XXH_OLD_NAMES - #define STRIPE_LEN XXH_STRIPE_LEN - #define ACC_NB XXH_ACC_NB - #endif - -XXH_FORCE_INLINE void XXH_writeLE64(void *dst, xxh_u64 v64) { - - if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); - memcpy(dst, &v64, sizeof(v64)); - -} - - /* Several intrinsic functions below are supposed to accept __int64 as - * argument, as documented in - * https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . - * However, several environments do not define __int64 type, - * requiring a workaround. - */ - #if !defined(__VMS) && \ - (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ - (__STDC_VERSION__ >= 199901L) /* C99 */)) -typedef int64_t xxh_i64; - #else -/* the following type must have a width of 64-bit */ -typedef long long xxh_i64; - #endif - - /* - * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the - * most optimized. - * - * It is a hardened version of UMAC, based off of FARSH's implementation. - * - * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD - * implementations, and it is ridiculously fast. - * - * We harden it by mixing the original input to the accumulators as well as - * the product. - * - * This means that in the (relatively likely) case of a multiply by zero, - * the original input is preserved. - * - * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve - * cross-pollination, as otherwise the upper and lower halves would be - * essentially independent. - * - * This doesn't matter on 64-bit hashes since they all get merged together - * in the end, so we skip the extra step. - * - * Both XXH3_64bits and XXH3_128bits use this subroutine. - */ - - #if (XXH_VECTOR == XXH_AVX512) || \ - (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) - - #ifndef XXH_TARGET_AVX512 - #define XXH_TARGET_AVX512 /* disable attribute target */ - #endif - -XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_accumulate_512_avx512( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, - const void *XXH_RESTRICT secret) { - - __m512i *const xacc = (__m512i *)acc; - XXH_ASSERT((((size_t)acc) & 63) == 0); - XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); - - { - - /* data_vec = input[0]; */ - __m512i const data_vec = _mm512_loadu_si512(input); - /* key_vec = secret[0]; */ - __m512i const key_vec = _mm512_loadu_si512(secret); - /* data_key = data_vec ^ key_vec; */ - __m512i const data_key = _mm512_xor_si512(data_vec, key_vec); - /* data_key_lo = data_key >> 32; */ - __m512i const data_key_lo = - _mm512_shuffle_epi32(data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); - /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ - __m512i const product = _mm512_mul_epu32(data_key, data_key_lo); - /* xacc[0] += swap(data_vec); */ - __m512i const data_swap = - _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); - __m512i const sum = _mm512_add_epi64(*xacc, data_swap); - /* xacc[0] += product; */ - *xacc = _mm512_add_epi64(product, sum); - - } - -} - -/* - * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. - * - * Multiplication isn't perfect, as explained by Google in HighwayHash: - * - * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to - * // varying degrees. In descending order of goodness, bytes - * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. - * // As expected, the upper and lower bytes are much worse. - * - * Source: - * https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 - * - * Since our algorithm uses a pseudorandom secret to add some variance into the - * mix, we don't need to (or want to) mix as often or as much as HighwayHash - * does. - * - * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid - * extraction. - * - * Both XXH3_64bits and XXH3_128bits use this subroutine. - */ - -XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_scrambleAcc_avx512( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT secret) { - - XXH_ASSERT((((size_t)acc) & 63) == 0); - XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); - { - - __m512i *const xacc = (__m512i *)acc; - const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); - - /* xacc[0] ^= (xacc[0] >> 47) */ - __m512i const acc_vec = *xacc; - __m512i const shifted = _mm512_srli_epi64(acc_vec, 47); - __m512i const data_vec = _mm512_xor_si512(acc_vec, shifted); - /* xacc[0] ^= secret; */ - __m512i const key_vec = _mm512_loadu_si512(secret); - __m512i const data_key = _mm512_xor_si512(data_vec, key_vec); - - /* xacc[0] *= XXH_PRIME32_1; */ - __m512i const data_key_hi = - _mm512_shuffle_epi32(data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); - __m512i const prod_lo = _mm512_mul_epu32(data_key, prime32); - __m512i const prod_hi = _mm512_mul_epu32(data_key_hi, prime32); - *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); - - } - -} - -XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512( - void *XXH_RESTRICT customSecret, xxh_u64 seed64) { - - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); - XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); - XXH_ASSERT(((size_t)customSecret & 63) == 0); - (void)(&XXH_writeLE64); - { - - int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); - __m512i const seed = _mm512_mask_set1_epi64( - _mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); - - const __m512i *const src = (const __m512i *)((const void *)XXH3_kSecret); - __m512i *const dest = (__m512i *)customSecret; - int i; - XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ - XXH_ASSERT(((size_t)dest & 63) == 0); - for (i = 0; i < nbRounds; ++i) { - - /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void - * const*', this will warn "discards 'const' qualifier". */ - union { - - const __m512i *cp; - void *p; - - } remote_const_void; - - remote_const_void.cp = src + i; - dest[i] = - _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); - - } - - } - -} - - #endif - - #if (XXH_VECTOR == XXH_AVX2) || \ - (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) - - #ifndef XXH_TARGET_AVX2 - #define XXH_TARGET_AVX2 /* disable attribute target */ - #endif - -XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, - const void *XXH_RESTRICT secret) { - - XXH_ASSERT((((size_t)acc) & 31) == 0); - { - - __m256i *const xacc = (__m256i *)acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. - */ - const __m256i *const xinput = (const __m256i *)input; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ - const __m256i *const xsecret = (const __m256i *)secret; - - size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { - - /* data_vec = xinput[i]; */ - __m256i const data_vec = _mm256_loadu_si256(xinput + i); - /* key_vec = xsecret[i]; */ - __m256i const key_vec = _mm256_loadu_si256(xsecret + i); - /* data_key = data_vec ^ key_vec; */ - __m256i const data_key = _mm256_xor_si256(data_vec, key_vec); - /* data_key_lo = data_key >> 32; */ - __m256i const data_key_lo = - _mm256_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); - /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ - __m256i const product = _mm256_mul_epu32(data_key, data_key_lo); - /* xacc[i] += swap(data_vec); */ - __m256i const data_swap = - _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); - __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); - /* xacc[i] += product; */ - xacc[i] = _mm256_add_epi64(product, sum); - - } - - } - -} - -XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_scrambleAcc_avx2( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT secret) { - - XXH_ASSERT((((size_t)acc) & 31) == 0); - { - - __m256i *const xacc = (__m256i *)acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ - const __m256i *const xsecret = (const __m256i *)secret; - const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); - - size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { - - /* xacc[i] ^= (xacc[i] >> 47) */ - __m256i const acc_vec = xacc[i]; - __m256i const shifted = _mm256_srli_epi64(acc_vec, 47); - __m256i const data_vec = _mm256_xor_si256(acc_vec, shifted); - /* xacc[i] ^= xsecret; */ - __m256i const key_vec = _mm256_loadu_si256(xsecret + i); - __m256i const data_key = _mm256_xor_si256(data_vec, key_vec); - - /* xacc[i] *= XXH_PRIME32_1; */ - __m256i const data_key_hi = - _mm256_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); - __m256i const prod_lo = _mm256_mul_epu32(data_key, prime32); - __m256i const prod_hi = _mm256_mul_epu32(data_key_hi, prime32); - xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); - - } - - } - -} - -XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2( - void *XXH_RESTRICT customSecret, xxh_u64 seed64) { - - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); - XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); - (void)(&XXH_writeLE64); - XXH_PREFETCH(customSecret); - { - - __m256i const seed = - _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, - (xxh_i64)(0U - seed64), (xxh_i64)seed64); - - const __m256i *const src = (const __m256i *)((const void *)XXH3_kSecret); - __m256i *dest = (__m256i *)customSecret; - - #if defined(__GNUC__) || defined(__clang__) - /* - * On GCC & Clang, marking 'dest' as modified will cause the compiler: - * - do not extract the secret from sse registers in the internal loop - * - use less common registers, and avoid pushing these reg into stack - */ - XXH_COMPILER_GUARD(dest); - #endif - XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ - XXH_ASSERT(((size_t)dest & 31) == 0); - - /* GCC -O2 need unroll loop manually */ - dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src + 0), seed); - dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src + 1), seed); - dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src + 2), seed); - dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src + 3), seed); - dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src + 4), seed); - dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src + 5), seed); - - } - -} - - #endif - - /* x86dispatch always generates SSE2 */ - #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) - - #ifndef XXH_TARGET_SSE2 - #define XXH_TARGET_SSE2 /* disable attribute target */ - #endif - -XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_accumulate_512_sse2( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, - const void *XXH_RESTRICT secret) { - - /* SSE2 is just a half-scale version of the AVX2 version. */ - XXH_ASSERT((((size_t)acc) & 15) == 0); - { - - __m128i *const xacc = (__m128i *)acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ - const __m128i *const xinput = (const __m128i *)input; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ - const __m128i *const xsecret = (const __m128i *)secret; - - size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { - - /* data_vec = xinput[i]; */ - __m128i const data_vec = _mm_loadu_si128(xinput + i); - /* key_vec = xsecret[i]; */ - __m128i const key_vec = _mm_loadu_si128(xsecret + i); - /* data_key = data_vec ^ key_vec; */ - __m128i const data_key = _mm_xor_si128(data_vec, key_vec); - /* data_key_lo = data_key >> 32; */ - __m128i const data_key_lo = - _mm_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); - /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ - __m128i const product = _mm_mul_epu32(data_key, data_key_lo); - /* xacc[i] += swap(data_vec); */ - __m128i const data_swap = - _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); - __m128i const sum = _mm_add_epi64(xacc[i], data_swap); - /* xacc[i] += product; */ - xacc[i] = _mm_add_epi64(product, sum); - - } - - } - -} - -XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_scrambleAcc_sse2( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT secret) { - - XXH_ASSERT((((size_t)acc) & 15) == 0); - { - - __m128i *const xacc = (__m128i *)acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ - const __m128i *const xsecret = (const __m128i *)secret; - const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); - - size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { - - /* xacc[i] ^= (xacc[i] >> 47) */ - __m128i const acc_vec = xacc[i]; - __m128i const shifted = _mm_srli_epi64(acc_vec, 47); - __m128i const data_vec = _mm_xor_si128(acc_vec, shifted); - /* xacc[i] ^= xsecret[i]; */ - __m128i const key_vec = _mm_loadu_si128(xsecret + i); - __m128i const data_key = _mm_xor_si128(data_vec, key_vec); - - /* xacc[i] *= XXH_PRIME32_1; */ - __m128i const data_key_hi = - _mm_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); - __m128i const prod_lo = _mm_mul_epu32(data_key, prime32); - __m128i const prod_hi = _mm_mul_epu32(data_key_hi, prime32); - xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); - - } - - } - -} - -XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2( - void *XXH_RESTRICT customSecret, xxh_u64 seed64) { - - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); - (void)(&XXH_writeLE64); - { - - int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); - - #if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 - /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ - XXH_ALIGN(16) - const xxh_i64 seed64x2[2] = {(xxh_i64)seed64, (xxh_i64)(0U - seed64)}; - __m128i const seed = _mm_load_si128((__m128i const *)seed64x2); - #else - __m128i const seed = - _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); - #endif - int i; - - const void *const src16 = XXH3_kSecret; - __m128i *dst16 = (__m128i *)customSecret; - #if defined(__GNUC__) || defined(__clang__) - /* - * On GCC & Clang, marking 'dest' as modified will cause the compiler: - * - do not extract the secret from sse registers in the internal loop - * - use less common registers, and avoid pushing these reg into stack - */ - XXH_COMPILER_GUARD(dst16); - #endif - XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ - XXH_ASSERT(((size_t)dst16 & 15) == 0); - - for (i = 0; i < nbRounds; ++i) { - - dst16[i] = - _mm_add_epi64(_mm_load_si128((const __m128i *)src16 + i), seed); - - } - - } - -} - - #endif - - #if (XXH_VECTOR == XXH_NEON) - -XXH_FORCE_INLINE void XXH3_accumulate_512_neon( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, - const void *XXH_RESTRICT secret) { - - XXH_ASSERT((((size_t)acc) & 15) == 0); - { - - uint64x2_t *const xacc = (uint64x2_t *)acc; - /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. - */ - uint8_t const *const xinput = (const uint8_t *)input; - uint8_t const *const xsecret = (const uint8_t *)secret; - - size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { - - /* data_vec = xinput[i]; */ - uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); - /* key_vec = xsecret[i]; */ - uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); - uint64x2_t data_key; - uint32x2_t data_key_lo, data_key_hi; - /* xacc[i] += swap(data_vec); */ - uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); - uint64x2_t const swapped = vextq_u64(data64, data64, 1); - xacc[i] = vaddq_u64(xacc[i], swapped); - /* data_key = data_vec ^ key_vec; */ - data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); - /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); - * data_key_hi = (uint32x2_t) (data_key >> 32); - * data_key = UNDEFINED; */ - XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); - /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ - xacc[i] = vmlal_u32(xacc[i], data_key_lo, data_key_hi); - - } - - } - -} - -XXH_FORCE_INLINE void XXH3_scrambleAcc_neon(void *XXH_RESTRICT acc, - const void *XXH_RESTRICT secret) { - - XXH_ASSERT((((size_t)acc) & 15) == 0); - - { - - uint64x2_t *xacc = (uint64x2_t *)acc; - uint8_t const *xsecret = (uint8_t const *)secret; - uint32x2_t prime = vdup_n_u32(XXH_PRIME32_1); - - size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { - - /* xacc[i] ^= (xacc[i] >> 47); */ - uint64x2_t acc_vec = xacc[i]; - uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); - uint64x2_t data_vec = veorq_u64(acc_vec, shifted); - - /* xacc[i] ^= xsecret[i]; */ - uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); - uint64x2_t data_key = veorq_u64(data_vec, vreinterpretq_u64_u8(key_vec)); - - /* xacc[i] *= XXH_PRIME32_1 */ - uint32x2_t data_key_lo, data_key_hi; - /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); - * data_key_hi = (uint32x2_t) (xacc[i] >> 32); - * xacc[i] = UNDEFINED; */ - XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); - { /* - * prod_hi = (data_key >> 32) * XXH_PRIME32_1; - * - * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will - * incorrectly "optimize" this: - * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); - * shifted = vshll_n_u32(tmp, 32); - * to this: - * tmp = "vmulq_u64"(a, b); // no such thing! - * shifted = vshlq_n_u64(tmp, 32); - * - * However, unlike SSE, Clang lacks a 64-bit multiply routine - * for NEON, and it scalarizes two 64-bit multiplies instead. - * - * vmull_u32 has the same timing as vmul_u32, and it avoids - * this bug completely. - * See https://bugs.llvm.org/show_bug.cgi?id=39967 - */ - uint64x2_t prod_hi = vmull_u32(data_key_hi, prime); - /* xacc[i] = prod_hi << 32; */ - xacc[i] = vshlq_n_u64(prod_hi, 32); - /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ - xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); - - } - - } - - } - -} - - #endif - - #if (XXH_VECTOR == XXH_VSX) - -XXH_FORCE_INLINE void XXH3_accumulate_512_vsx(void *XXH_RESTRICT acc, - const void *XXH_RESTRICT input, - const void *XXH_RESTRICT secret) { - - xxh_u64x2 *const xacc = (xxh_u64x2 *)acc; /* presumed aligned */ - xxh_u64x2 const *const xinput = - (xxh_u64x2 const *)input; /* no alignment restriction */ - xxh_u64x2 const *const xsecret = - (xxh_u64x2 const *)secret; /* no alignment restriction */ - xxh_u64x2 const v32 = {32, 32}; - size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { - - /* data_vec = xinput[i]; */ - xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); - /* key_vec = xsecret[i]; */ - xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); - xxh_u64x2 const data_key = data_vec ^ key_vec; - /* shuffled = (data_key << 32) | (data_key >> 32); */ - xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); - /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & - * 0xFFFFFFFF); */ - xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); - xacc[i] += product; - - /* swap high and low halves */ - #ifdef __s390x__ - xacc[i] += vec_permi(data_vec, data_vec, 2); - #else - xacc[i] += vec_xxpermdi(data_vec, data_vec, 2); - #endif - - } - -} - -XXH_FORCE_INLINE void XXH3_scrambleAcc_vsx(void *XXH_RESTRICT acc, - const void *XXH_RESTRICT secret) { - - XXH_ASSERT((((size_t)acc) & 15) == 0); - - { - - xxh_u64x2 *const xacc = (xxh_u64x2 *)acc; - const xxh_u64x2 *const xsecret = (const xxh_u64x2 *)secret; - /* constants */ - xxh_u64x2 const v32 = {32, 32}; - xxh_u64x2 const v47 = {47, 47}; - xxh_u32x4 const prime = {XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, - XXH_PRIME32_1}; - size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { - - /* xacc[i] ^= (xacc[i] >> 47); */ - xxh_u64x2 const acc_vec = xacc[i]; - xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); - - /* xacc[i] ^= xsecret[i]; */ - xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); - xxh_u64x2 const data_key = data_vec ^ key_vec; - - /* xacc[i] *= XXH_PRIME32_1 */ - /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & - * 0xFFFFFFFF); */ - xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); - /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ - xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); - xacc[i] = prod_odd + (prod_even << v32); - - } - - } - -} - - #endif - -/* scalar variants - universal */ - -XXH_FORCE_INLINE void XXH3_accumulate_512_scalar( - void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, - const void *XXH_RESTRICT secret) { - - xxh_u64 *const xacc = (xxh_u64 *)acc; /* presumed aligned */ - const xxh_u8 *const xinput = - (const xxh_u8 *)input; /* no alignment restriction */ - const xxh_u8 *const xsecret = - (const xxh_u8 *)secret; /* no alignment restriction */ - size_t i; - XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN - 1)) == 0); - for (i = 0; i < XXH_ACC_NB; i++) { - - xxh_u64 const data_val = XXH_readLE64(xinput + 8 * i); - xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i * 8); - xacc[i ^ 1] += data_val; /* swap adjacent lanes */ - xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); - - } - -} - -XXH_FORCE_INLINE void XXH3_scrambleAcc_scalar(void *XXH_RESTRICT acc, - const void *XXH_RESTRICT secret) { - - xxh_u64 *const xacc = (xxh_u64 *)acc; /* presumed aligned */ - const xxh_u8 *const xsecret = - (const xxh_u8 *)secret; /* no alignment restriction */ - size_t i; - XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN - 1)) == 0); - for (i = 0; i < XXH_ACC_NB; i++) { - - xxh_u64 const key64 = XXH_readLE64(xsecret + 8 * i); - xxh_u64 acc64 = xacc[i]; - acc64 = XXH_xorshift64(acc64, 47); - acc64 ^= key64; - acc64 *= XXH_PRIME32_1; - xacc[i] = acc64; - - } - -} - -XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( - void *XXH_RESTRICT customSecret, xxh_u64 seed64) { - - /* - * We need a separate pointer for the hack below, - * which requires a non-const pointer. - * Any decent compiler will optimize this out otherwise. - */ - const xxh_u8 *kSecretPtr = XXH3_kSecret; - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); - - #if defined(__clang__) && defined(__aarch64__) - /* - * UGLY HACK: - * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are - * placed sequentially, in order, at the top of the unrolled loop. - * - * While MOVK is great for generating constants (2 cycles for a 64-bit - * constant compared to 4 cycles for LDR), long MOVK chains stall the - * integer pipelines: - * I L S - * MOVK - * MOVK - * MOVK - * MOVK - * ADD - * SUB STR - * STR - * By forcing loads from memory (as the asm line causes Clang to assume - * that XXH3_kSecretPtr has been changed), the pipelines are used more - * efficiently: - * I L S - * LDR - * ADD LDR - * SUB STR - * STR - * XXH3_64bits_withSeed, len == 256, Snapdragon 835 - * without hack: 2654.4 MB/s - * with hack: 3202.9 MB/s - */ - XXH_COMPILER_GUARD(kSecretPtr); - #endif - /* - * Note: in debug mode, this overrides the asm optimization - * and Clang will emit MOVK chains again. - */ - XXH_ASSERT(kSecretPtr == XXH3_kSecret); - - { - - int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; - int i; - for (i = 0; i < nbRounds; i++) { - - /* - * The asm hack causes Clang to assume that kSecretPtr aliases with - * customSecret, and on aarch64, this prevented LDP from merging two - * loads together for free. Putting the loads together before the stores - * properly generates LDP. - */ - xxh_u64 lo = XXH_readLE64(kSecretPtr + 16 * i) + seed64; - xxh_u64 hi = XXH_readLE64(kSecretPtr + 16 * i + 8) - seed64; - XXH_writeLE64((xxh_u8 *)customSecret + 16 * i, lo); - XXH_writeLE64((xxh_u8 *)customSecret + 16 * i + 8, hi); - - } - - } - -} - -typedef void (*XXH3_f_accumulate_512)(void *XXH_RESTRICT, const void *, - const void *); -typedef void (*XXH3_f_scrambleAcc)(void *XXH_RESTRICT, const void *); -typedef void (*XXH3_f_initCustomSecret)(void *XXH_RESTRICT, xxh_u64); - - #if (XXH_VECTOR == XXH_AVX512) - - #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 - #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 - #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 - - #elif (XXH_VECTOR == XXH_AVX2) - - #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 - #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 - #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 - - #elif (XXH_VECTOR == XXH_SSE2) - - #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 - #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 - #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 - - #elif (XXH_VECTOR == XXH_NEON) - - #define XXH3_accumulate_512 XXH3_accumulate_512_neon - #define XXH3_scrambleAcc XXH3_scrambleAcc_neon - #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - - #elif (XXH_VECTOR == XXH_VSX) - - #define XXH3_accumulate_512 XXH3_accumulate_512_vsx - #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx - #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - - #else /* scalar */ - - #define XXH3_accumulate_512 XXH3_accumulate_512_scalar - #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar - #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - - #endif - - #ifndef XXH_PREFETCH_DIST - #ifdef __clang__ - #define XXH_PREFETCH_DIST 320 - #else - #if (XXH_VECTOR == XXH_AVX512) - #define XXH_PREFETCH_DIST 512 - #else - #define XXH_PREFETCH_DIST 384 - #endif - #endif /* __clang__ */ - #endif /* XXH_PREFETCH_DIST */ - -/* - * XXH3_accumulate() - * Loops over XXH3_accumulate_512(). - * Assumption: nbStripes will not overflow the secret size - */ -XXH_FORCE_INLINE void XXH3_accumulate(xxh_u64 *XXH_RESTRICT acc, - const xxh_u8 *XXH_RESTRICT input, - const xxh_u8 *XXH_RESTRICT secret, - size_t nbStripes, - XXH3_f_accumulate_512 f_acc512) { - - size_t n; - for (n = 0; n < nbStripes; n++) { - - const xxh_u8 *const in = input + n * XXH_STRIPE_LEN; - XXH_PREFETCH(in + XXH_PREFETCH_DIST); - f_acc512(acc, in, secret + n * XXH_SECRET_CONSUME_RATE); - - } - -} - -XXH_FORCE_INLINE void XXH3_hashLong_internal_loop( - xxh_u64 *XXH_RESTRICT acc, const xxh_u8 *XXH_RESTRICT input, size_t len, - const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { - - size_t const nbStripesPerBlock = - (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; - size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; - size_t const nb_blocks = (len - 1) / block_len; - - size_t n; - - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - - for (n = 0; n < nb_blocks; n++) { - - XXH3_accumulate(acc, input + n * block_len, secret, nbStripesPerBlock, - f_acc512); - f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); - - } - - /* last partial block */ - XXH_ASSERT(len > XXH_STRIPE_LEN); - { - - size_t const nbStripes = - ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; - XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); - XXH3_accumulate(acc, input + nb_blocks * block_len, secret, nbStripes, - f_acc512); - - /* last stripe */ - { - - const xxh_u8 *const p = input + len - XXH_STRIPE_LEN; - #define XXH_SECRET_LASTACC_START \ - 7 /* not aligned on 8, last secret is different from acc & scrambler \ - */ - f_acc512(acc, p, - secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); - - } - - } - -} - -XXH_FORCE_INLINE xxh_u64 XXH3_mix2Accs(const xxh_u64 *XXH_RESTRICT acc, - const xxh_u8 *XXH_RESTRICT secret) { - - return XXH3_mul128_fold64(acc[0] ^ XXH_readLE64(secret), - acc[1] ^ XXH_readLE64(secret + 8)); - -} - -static XXH64_hash_t XXH3_mergeAccs(const xxh_u64 *XXH_RESTRICT acc, - const xxh_u8 *XXH_RESTRICT secret, - xxh_u64 start) { - - xxh_u64 result64 = start; - size_t i = 0; - - for (i = 0; i < 4; i++) { - - result64 += XXH3_mix2Accs(acc + 2 * i, secret + 16 * i); - #if defined(__clang__) /* Clang */ \ - && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ - /* - * UGLY HACK: - * Prevent autovectorization on Clang ARMv7-a. Exact same problem as - * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. - * XXH3_64bits, len == 256, Snapdragon 835: - * without hack: 2063.7 MB/s - * with hack: 2560.7 MB/s - */ - XXH_COMPILER_GUARD(result64); - #endif - - } - - return XXH3_avalanche(result64); - -} - - #define XXH3_INIT_ACC \ - { \ - \ - XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ - XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 \ - \ - } - -XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal( - const void *XXH_RESTRICT input, size_t len, const void *XXH_RESTRICT secret, - size_t secretSize, XXH3_f_accumulate_512 f_acc512, - XXH3_f_scrambleAcc f_scramble) { - - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; - - XXH3_hashLong_internal_loop(acc, (const xxh_u8 *)input, len, - (const xxh_u8 *)secret, secretSize, f_acc512, - f_scramble); - - /* converge into final hash */ - XXH_STATIC_ASSERT(sizeof(acc) == 64); - /* do not align on 8, so that the secret is different from the accumulator - */ - #define XXH_SECRET_MERGEACCS_START 11 - XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); - return XXH3_mergeAccs(acc, - (const xxh_u8 *)secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)len * XXH_PRIME64_1); - -} - -/* - * It's important for performance that XXH3_hashLong is not inlined. - */ -XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_withSecret( - const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, - const xxh_u8 *XXH_RESTRICT secret, size_t secretLen) { - - (void)seed64; - return XXH3_hashLong_64b_internal(input, len, secret, secretLen, - XXH3_accumulate_512, XXH3_scrambleAcc); - -} - -/* - * It's important for performance that XXH3_hashLong is not inlined. - * Since the function is not inlined, the compiler may not be able to understand - * that, in some scenarios, its `secret` argument is actually a compile time - * constant. This variant enforces that the compiler can detect that, and uses - * this opportunity to streamline the generated code for better performance. - */ -XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_default( - const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, - const xxh_u8 *XXH_RESTRICT secret, size_t secretLen) { - - (void)seed64; - (void)secret; - (void)secretLen; - return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, - sizeof(XXH3_kSecret), XXH3_accumulate_512, - XXH3_scrambleAcc); - -} - -/* - * XXH3_hashLong_64b_withSeed(): - * Generate a custom key based on alteration of default XXH3_kSecret with the - * seed, and then use this key for long mode hashing. - * - * This operation is decently fast but nonetheless costs a little bit of time. - * Try to avoid it whenever possible (typically when seed==0). - * - * It's important for performance that XXH3_hashLong is not inlined. Not sure - * why (uop cache maybe?), but the difference is large and easily measurable. - */ -XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed_internal( - const void *input, size_t len, XXH64_hash_t seed, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble, - XXH3_f_initCustomSecret f_initSec) { - - if (seed == 0) - return XXH3_hashLong_64b_internal( - input, len, XXH3_kSecret, sizeof(XXH3_kSecret), f_acc512, f_scramble); - { - - XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; - f_initSec(secret, seed); - return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), - f_acc512, f_scramble); - - } - -} - -/* - * It's important for performance that XXH3_hashLong is not inlined. - */ -XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed(const void *input, - size_t len, - XXH64_hash_t seed, - const xxh_u8 *secret, - size_t secretLen) { - - (void)secret; - (void)secretLen; - return XXH3_hashLong_64b_withSeed_internal( - input, len, seed, XXH3_accumulate_512, XXH3_scrambleAcc, - XXH3_initCustomSecret); - -} - -typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void *XXH_RESTRICT, size_t, - XXH64_hash_t, - const xxh_u8 *XXH_RESTRICT, size_t); - -XXH_FORCE_INLINE XXH64_hash_t -XXH3_64bits_internal(const void *XXH_RESTRICT input, size_t len, - XXH64_hash_t seed64, const void *XXH_RESTRICT secret, - size_t secretLen, XXH3_hashLong64_f f_hashLong) { - - XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); - /* - * If an action is to be taken if `secretLen` condition is not respected, - * it should be done here. - * For now, it's a contract pre-condition. - * Adding a check and a branch here would cost performance at every hash. - * Also, note that function signature doesn't offer room to return an error. - */ - if (len <= 16) - return XXH3_len_0to16_64b((const xxh_u8 *)input, len, - (const xxh_u8 *)secret, seed64); - if (len <= 128) - return XXH3_len_17to128_64b((const xxh_u8 *)input, len, - (const xxh_u8 *)secret, secretLen, seed64); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_64b((const xxh_u8 *)input, len, - (const xxh_u8 *)secret, secretLen, seed64); - return f_hashLong(input, len, seed64, (const xxh_u8 *)secret, secretLen); - -} - -/* === Public entry point === */ - -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *input, size_t len) { - - return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), - XXH3_hashLong_64b_default); - + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *input, - size_t len, - const void *secret, - size_t secretSize) { - return XXH3_64bits_internal(input, len, 0, secret, secretSize, - XXH3_hashLong_64b_withSecret); +/* ======= Long Keys ======= */ -} +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void *input, size_t len, - XXH64_hash_t seed) { +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif - return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, - sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ -} +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif -/* === XXH3 streaming === */ /* - * Malloc's a pointer that is always aligned to align. + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. * - * This must be freed with `XXH_alignedFree()`. + * It is a hardened version of UMAC, based off of FARSH's implementation. * - * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte - * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 - * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. * - * This underalignment previously caused a rather obvious crash which went - * completely unnoticed due to XXH3_createState() not actually being tested. - * Credit to RedSpah for noticing this bug. + * We harden it by mixing the original input to the accumulators as well as the product. * - * The alignment is done manually: Functions like posix_memalign or _mm_malloc - * are avoided: To maintain portability, we would have to write a fallback - * like this anyways, and besides, testing for the existence of library - * functions without relying on external build tools is impossible. + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. * - * The method is simple: Overallocate, manually align, and store the offset - * to the original behind the returned pointer. + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. * - * Align must be a power of 2 and 8 <= align <= 128. + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. */ -static void *XXH_alignedMalloc(size_t s, size_t align) { - - XXH_ASSERT(align <= 128 && align >= 8); /* range check */ - XXH_ASSERT((align & (align - 1)) == 0); /* power of 2 */ - XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ - { /* Overallocate to make room for manual realignment and an offset byte */ - xxh_u8 *base = (xxh_u8 *)XXH_malloc(s + align); - if (base != NULL) { - /* - * Get the offset needed to align this pointer. - * - * Even if the returned pointer is aligned, there will always be - * at least one byte to store the offset to the original pointer. - */ - size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ - /* Add the offset for the now-aligned pointer */ - xxh_u8 *ptr = base + offset; +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) - XXH_ASSERT((size_t)ptr % align == 0); +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif - /* Store the offset immediately before the returned pointer. */ - ptr[-1] = (xxh_u8)offset; - return ptr; +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); } - - return NULL; - - } - } +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) /* - * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass - * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. */ -static void XXH_alignedFree(void *p) { - - if (p != NULL) { - - xxh_u8 *ptr = (xxh_u8 *)p; - /* Get the offset byte we added in XXH_malloc. */ - xxh_u8 offset = ptr[-1]; - /* Free the original malloc'd pointer */ - xxh_u8 *base = ptr - offset; - XXH_free(base); - - } +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void) { - - XXH3_state_t *const state = - (XXH3_state_t *)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); - if (state == NULL) return NULL; - XXH3_INITSTATE(state); - return state; +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr) { - - XXH_alignedFree(statePtr); - return XXH_OK; - -} +#endif -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t *dst_state, - const XXH3_state_t *src_state) { +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) - memcpy(dst_state, src_state, sizeof(*dst_state)); +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } } -static void XXH3_reset_internal(XXH3_state_t *statePtr, XXH64_hash_t seed, - const void *secret, size_t secretSize) { +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); - size_t const initStart = offsetof(XXH3_state_t, bufferedSize); - size_t const initLength = - offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; - XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); - XXH_ASSERT(statePtr != NULL); - /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ - memset((char *)statePtr + initStart, 0, initLength); - statePtr->acc[0] = XXH_PRIME32_3; - statePtr->acc[1] = XXH_PRIME64_1; - statePtr->acc[2] = XXH_PRIME64_2; - statePtr->acc[3] = XXH_PRIME64_3; - statePtr->acc[4] = XXH_PRIME64_4; - statePtr->acc[5] = XXH_PRIME32_2; - statePtr->acc[6] = XXH_PRIME64_5; - statePtr->acc[7] = XXH_PRIME32_1; - statePtr->seed = seed; - statePtr->extSecret = (const unsigned char *)secret; - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; - statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t *statePtr) { - - if (statePtr == NULL) return XXH_ERROR; - XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; - -} +#endif -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret( - XXH3_state_t *statePtr, const void *secret, size_t secretSize) { +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) - if (statePtr == NULL) return XXH_ERROR; - XXH3_reset_internal(statePtr, 0, secret, secretSize); - if (secret == NULL) return XXH_ERROR; - if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; - return XXH_OK; +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t *statePtr, - XXH64_hash_t seed) { - - if (statePtr == NULL) return XXH_ERROR; - if (seed == 0) return XXH3_64bits_reset(statePtr); - if (seed != statePtr->seed) - XXH3_initCustomSecret(statePtr->customSecret, seed); - XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } } -/* Note : when XXH3_consumeStripes() is invoked, - * there must be a guarantee that at least one more byte must be consumed from - * input - * so that the function can blindly consume all stripes using the "normal" - * secret segment */ -XXH_FORCE_INLINE void XXH3_consumeStripes( - xxh_u64 *XXH_RESTRICT acc, size_t *XXH_RESTRICT nbStripesSoFarPtr, - size_t nbStripesPerBlock, const xxh_u8 *XXH_RESTRICT input, - size_t nbStripes, const xxh_u8 *XXH_RESTRICT secret, size_t secretLimit, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { - - XXH_ASSERT(nbStripes <= - nbStripesPerBlock); /* can handle max 1 scramble per invocation */ - XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); - if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { - - /* need a scrambling operation */ - size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; - size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; - XXH3_accumulate(acc, input, - secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, - nbStripesToEndofBlock, f_acc512); - f_scramble(acc, secret + secretLimit); - XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, - nbStripesAfterBlock, f_acc512); - *nbStripesSoFarPtr = nbStripesAfterBlock; - - } else { +#endif - XXH3_accumulate(acc, input, - secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, - nbStripes, f_acc512); - *nbStripesSoFarPtr += nbStripes; +#if (XXH_VECTOR == XXH_NEON) - } +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); -} +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); -/* - * Both XXH3_64bits_update and XXH3_128bits_update use this routine. +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. */ -XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, - const xxh_u8 *input, size_t len, - XXH3_f_accumulate_512 f_acc512, - XXH3_f_scrambleAcc f_scramble) { - - if (input == NULL) - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) - return XXH_OK; - #else - return XXH_ERROR; - #endif - - { - - const xxh_u8 *const bEnd = input + len; - const unsigned char *const secret = - (state->extSecret == NULL) ? state->customSecret : state->extSecret; - - state->totalLen += len; - XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); - - if (state->bufferedSize + len <= - XXH3_INTERNALBUFFER_SIZE) { /* fill in tmp buffer */ - XXH_memcpy(state->buffer + state->bufferedSize, input, len); - state->bufferedSize += (XXH32_hash_t)len; - return XXH_OK; +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) - /* total input is now > XXH3_INTERNALBUFFER_SIZE */ - - #define XXH3_INTERNALBUFFER_STRIPES \ - (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) - XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == - 0); /* clean multiple */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); - /* - * Internal buffer is partially filled (always, except at beginning) - * Complete it, then consume it. - */ - if (state->bufferedSize) { + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; - size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; - XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); - input += loadSize; - XXH3_consumeStripes(state->acc, &state->nbStripesSoFar, - state->nbStripesPerBlock, state->buffer, - XXH3_INTERNALBUFFER_STRIPES, secret, - state->secretLimit, f_acc512, f_scramble); - state->bufferedSize = 0; + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } } +} +#endif - XXH_ASSERT(input < bEnd); - - /* Consume input by a multiple of internal buffer size */ - if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { - - const xxh_u8 *const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; - do { - - XXH3_consumeStripes(state->acc, &state->nbStripesSoFar, - state->nbStripesPerBlock, input, - XXH3_INTERNALBUFFER_STRIPES, secret, - state->secretLimit, f_acc512, f_scramble); - input += XXH3_INTERNALBUFFER_SIZE; - - } while (input < limit); - - /* for last partial stripe */ - memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, - input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); - +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; } - - XXH_ASSERT(input < bEnd); - - /* Some remaining input (always) : buffer it */ - XXH_memcpy(state->buffer, input, (size_t)(bEnd - input)); - state->bufferedSize = (XXH32_hash_t)(bEnd - input); - - } - - return XXH_OK; - } - -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update(XXH3_state_t *state, - const void *input, size_t len) { - - return XXH3_update(state, (const xxh_u8 *)input, len, XXH3_accumulate_512, - XXH3_scrambleAcc); - +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } } -XXH_FORCE_INLINE void XXH3_digest_long(XXH64_hash_t *acc, - const XXH3_state_t *state, - const unsigned char *secret) { - - /* - * Digest on a local copy. This way, the state remains unaltered, and it can - * continue ingesting more input afterwards. - */ - memcpy(acc, state->acc, sizeof(state->acc)); - if (state->bufferedSize >= XXH_STRIPE_LEN) { - - size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; - size_t nbStripesSoFar = state->nbStripesSoFar; - XXH3_consumeStripes(acc, &nbStripesSoFar, state->nbStripesPerBlock, - state->buffer, nbStripes, secret, state->secretLimit, - XXH3_accumulate_512, XXH3_scrambleAcc); - /* last stripe */ - XXH3_accumulate_512(acc, - state->buffer + state->bufferedSize - XXH_STRIPE_LEN, - secret + state->secretLimit - XXH_SECRET_LASTACC_START); - - } else { /* bufferedSize < XXH_STRIPE_LEN */ - - xxh_u8 lastStripe[XXH_STRIPE_LEN]; - size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; - XXH_ASSERT(state->bufferedSize > - 0); /* there is always some input buffered */ - memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, - catchupSize); - memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); - XXH3_accumulate_512(acc, lastStripe, - secret + state->secretLimit - XXH_SECRET_LASTACC_START); - - } +#endif +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *state) { - - const unsigned char *const secret = - (state->extSecret == NULL) ? state->customSecret : state->extSecret; - if (state->totalLen > XXH3_MIDSIZE_MAX) { - - XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; - XXH3_digest_long(acc, state, secret); - return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)state->totalLen * XXH_PRIME64_1); - - } - - /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ - if (state->seed) - return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, - state->seed); - return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), - secret, state->secretLimit + XXH_STRIPE_LEN); - +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } } - #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) - -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API void XXH3_generateSecret(void *secretBuffer, - const void *customSeed, - size_t customSeedSize) { - - XXH_ASSERT(secretBuffer != NULL); - if (customSeedSize == 0) { - - memcpy(secretBuffer, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); - return; - - } - - XXH_ASSERT(customSeed != NULL); +#endif - { +/* scalar variants - universal */ - size_t const segmentSize = sizeof(XXH128_hash_t); - size_t const nbSegments = XXH_SECRET_DEFAULT_SIZE / segmentSize; - XXH128_canonical_t scrambler; - XXH64_hash_t seeds[12]; - size_t segnb; - XXH_ASSERT(nbSegments == 12); - XXH_ASSERT(segmentSize * nbSegments == - XXH_SECRET_DEFAULT_SIZE); /* exact multiple */ - XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif - /* - * Copy customSeed to seeds[], truncating or repeating as necessary. - */ +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); { - - size_t toFill = XXH_MIN(customSeedSize, sizeof(seeds)); - size_t filled = toFill; - memcpy(seeds, customSeed, toFill); - while (filled < sizeof(seeds)) { - - toFill = XXH_MIN(filled, sizeof(seeds) - filled); - memcpy((char *)seeds + filled, seeds, toFill); - filled += toFill; - - } - + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); } +} - /* generate secret */ - memcpy(secretBuffer, &scrambler, sizeof(scrambler)); - for (segnb = 1; segnb < nbSegments; segnb++) { - - size_t const segmentStart = segnb * segmentSize; - XXH128_canonical_t segment; - XXH128_canonicalFromHash(&segment, - XXH128(&scrambler, sizeof(scrambler), - XXH_readLE64(seeds + segnb) + segnb)); - memcpy((char *)secretBuffer + segmentStart, &segment, sizeof(segment)); - +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); } - - } - } +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) -/* ========================================== - * XXH3 128 bits (a.k.a XXH128) - * ========================================== - * XXH3's 128-bit variant has better mixing and strength than the 64-bit - * variant, even without counting the significantly larger output size. - * - * For example, extra steps are taken to avoid the seed-dependent collisions - * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). - * - * This strength naturally comes at the cost of some speed, especially on short - * lengths. Note that longer hashes are about as fast as the 64-bit version - * due to it using only a slight modification of the 64-bit loop. +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). * - * XXH128 is also more oriented towards 64-bit machines. It is still extremely - * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. */ - -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_1to3_128b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { - - /* A doubled version of 1to3_64b with different constants. */ - XXH_ASSERT(input != NULL); - XXH_ASSERT(1 <= len && len <= 3); - XXH_ASSERT(secret != NULL); - /* - * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } - * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } - * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } - */ - { - - xxh_u8 const c1 = input[0]; - xxh_u8 const c2 = input[len >> 1]; - xxh_u8 const c3 = input[len - 1]; - xxh_u32 const combinedl = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) | - ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); - xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); - xxh_u64 const bitflipl = - (XXH_readLE32(secret) ^ XXH_readLE32(secret + 4)) + seed; - xxh_u64 const bitfliph = - (XXH_readLE32(secret + 8) ^ XXH_readLE32(secret + 12)) - seed; - xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; - xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; - XXH128_hash_t h128; - h128.low64 = XXH64_avalanche(keyed_lo); - h128.high64 = XXH64_avalanche(keyed_hi); - return h128; - - } - -} - -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_4to8_128b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { - - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(4 <= len && len <= 8); - seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; - { - - xxh_u32 const input_lo = XXH_readLE32(input); - xxh_u32 const input_hi = XXH_readLE32(input + len - 4); - xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); - xxh_u64 const bitflip = - (XXH_readLE64(secret + 16) ^ XXH_readLE64(secret + 24)) + seed; - xxh_u64 const keyed = input_64 ^ bitflip; - - /* Shift len to the left to ensure it is even, this avoids even multiplies. - */ - XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); - - m128.high64 += (m128.low64 << 1); - m128.low64 ^= (m128.high64 >> 3); - - m128.low64 = XXH_xorshift64(m128.low64, 35); - m128.low64 *= 0x9FB21C651E98DF25ULL; - m128.low64 = XXH_xorshift64(m128.low64, 28); - m128.high64 = XXH3_avalanche(m128.high64); - return m128; - - } - +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } } -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_9to16_128b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { - - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(9 <= len && len <= 16); - { +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} - xxh_u64 const bitflipl = - (XXH_readLE64(secret + 32) ^ XXH_readLE64(secret + 40)) - seed; - xxh_u64 const bitfliph = - (XXH_readLE64(secret + 48) ^ XXH_readLE64(secret + 56)) + seed; - xxh_u64 const input_lo = XXH_readLE64(input); - xxh_u64 input_hi = XXH_readLE64(input + len - 8); - XXH128_hash_t m128 = - XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ /* - * Put len in the middle of m128 to ensure that the length gets mixed to - * both the low and high bits in the 128x64 multiply below. + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. */ - m128.low64 += (xxh_u64)(len - 1) << 54; - input_hi ^= bitfliph; + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) /* - * Add the high 32 bits of input_hi to the high 32 bits of m128, then - * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to - * the high 64 bits of m128. + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. * - * The best approach to this operation is different on 32-bit and 64-bit. + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s */ - if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ - /* - * 32-bit optimized version, which is more readable. - * - * On 32-bit, it removes an ADC and delays a dependency between the two - * halves of m128.high64, but it generates an extra mask on 64-bit. - */ - m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + - XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); - - } else { - - /* - * 64-bit optimized (albeit more confusing) version. - * - * Uses some properties of addition and multiplication to remove the mask: - * - * Let: - * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) - * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) - * c = XXH_PRIME32_2 - * - * a + (b * c) - * Inverse Property: x + y - x == y - * a + (b * (1 + c - 1)) - * Distributive Property: x * (y + z) == (x * y) + (x * z) - * a + (b * 1) + (b * (c - 1)) - * Identity Property: x * 1 == x - * a + b + (b * (c - 1)) - * - * Substitute a, b, and c: - * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - - * 1)) - * - * Since input_hi.hi + input_hi.lo == input_hi, we get this: - * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) - */ - m128.high64 += - input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); - - } - - /* m128 ^= XXH_swap64(m128 >> 64); */ - m128.low64 ^= XXH_swap64(m128.high64); - - { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ - XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); - h128.high64 += m128.high64 * XXH_PRIME64_2; - - h128.low64 = XXH3_avalanche(h128.low64); - h128.high64 = XXH3_avalanche(h128.high64); - return h128; - - } - - } - + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } } -/* - * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN - */ -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_0to16_128b(const xxh_u8 *input, - size_t len, - const xxh_u8 *secret, - XXH64_hash_t seed) { - - XXH_ASSERT(len <= 16); - { - if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); - if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); - if (len) return XXH3_len_1to3_128b(input, len, secret, seed); - { - - XXH128_hash_t h128; - xxh_u64 const bitflipl = - XXH_readLE64(secret + 64) ^ XXH_readLE64(secret + 72); - xxh_u64 const bitfliph = - XXH_readLE64(secret + 80) ^ XXH_readLE64(secret + 88); - h128.low64 = XXH64_avalanche(seed ^ bitflipl); - h128.high64 = XXH64_avalanche(seed ^ bitfliph); - return h128; - - } +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); - } -} +#if (XXH_VECTOR == XXH_AVX512) -/* - * A bit slower than XXH3_mix16B, but handles multiply by zero better. - */ -XXH_FORCE_INLINE XXH128_hash_t XXH128_mix32B(XXH128_hash_t acc, - const xxh_u8 *input_1, - const xxh_u8 *input_2, - const xxh_u8 *secret, - XXH64_hash_t seed) { +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 - acc.low64 += XXH3_mix16B(input_1, secret + 0, seed); - acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); - acc.high64 += XXH3_mix16B(input_2, secret + 16, seed); - acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); - return acc; +#elif (XXH_VECTOR == XXH_AVX2) -} +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 -XXH_FORCE_INLINE XXH128_hash_t XXH3_len_17to128_128b( - const xxh_u8 *XXH_RESTRICT input, size_t len, - const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { +#elif (XXH_VECTOR == XXH_SSE2) - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - (void)secretSize; - XXH_ASSERT(16 < len && len <= 128); +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 - { +#elif (XXH_VECTOR == XXH_NEON) - XXH128_hash_t acc; - acc.low64 = len * XXH_PRIME64_1; - acc.high64 = 0; - if (len > 32) { +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - if (len > 64) { +#elif (XXH_VECTOR == XXH_VSX) - if (len > 96) { +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - acc = XXH128_mix32B(acc, input + 48, input + len - 64, secret + 96, - seed); +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - } +#else /* scalar */ - acc = - XXH128_mix32B(acc, input + 32, input + len - 48, secret + 64, seed); +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - } +#endif - acc = XXH128_mix32B(acc, input + 16, input + len - 32, secret + 32, seed); +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif - } +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; - acc = XXH128_mix32B(acc, input, input + len - 16, secret, seed); - { + size_t n; - XXH128_hash_t h128; - h128.low64 = acc.low64 + acc.high64; - h128.high64 = (acc.low64 * XXH_PRIME64_1) + (acc.high64 * XXH_PRIME64_4) + - ((len - seed) * XXH_PRIME64_2); - h128.low64 = XXH3_avalanche(h128.low64); - h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); - return h128; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); } - } + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } } -XXH_NO_INLINE XXH128_hash_t XXH3_len_129to240_128b( - const xxh_u8 *XXH_RESTRICT input, size_t len, - const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { - - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - (void)secretSize; - XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} - { +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; - XXH128_hash_t acc; - int const nbRounds = (int)len / 32; - int i; - acc.low64 = len * XXH_PRIME64_1; - acc.high64 = 0; for (i = 0; i < 4; i++) { - - acc = XXH128_mix32B(acc, input + (32 * i), input + (32 * i) + 16, - secret + (32 * i), seed); - + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif } - acc.low64 = XXH3_avalanche(acc.low64); - acc.high64 = XXH3_avalanche(acc.high64); - XXH_ASSERT(nbRounds >= 4); - for (i = 4; i < nbRounds; i++) { - - acc = XXH128_mix32B(acc, input + (32 * i), input + (32 * i) + 16, - secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), - seed); + return XXH3_avalanche(result64); +} - } +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } - /* last bytes */ - acc = XXH128_mix32B( - acc, input + len - 16, input + len - 32, - secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, - 0ULL - seed); +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; - { + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); - XXH128_hash_t h128; - h128.low64 = acc.low64 + acc.high64; - h128.high64 = (acc.low64 * XXH_PRIME64_1) + (acc.high64 * XXH_PRIME64_4) + - ((len - seed) * XXH_PRIME64_2); - h128.low64 = XXH3_avalanche(h128.low64); - h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); - return h128; + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} - } +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} - } +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); + } } -XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_internal( - const void *XXH_RESTRICT input, size_t len, - const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; - XXH3_hashLong_internal_loop(acc, (const xxh_u8 *)input, len, secret, - secretSize, f_acc512, f_scramble); +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); - /* converge into final hash */ - XXH_STATIC_ASSERT(sizeof(acc) == 64); - XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); - { +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} - XXH128_hash_t h128; - h128.low64 = XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)len * XXH_PRIME64_1); - h128.high64 = XXH3_mergeAccs( - acc, secret + secretSize - sizeof(acc) - XXH_SECRET_MERGEACCS_START, - ~((xxh_u64)len * XXH_PRIME64_2)); - return h128; - } +/* === Public entry point === */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) +{ + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); } -/* - * It's important for performance that XXH3_hashLong is not inlined. - */ -XXH_NO_INLINE XXH128_hash_t XXH3_hashLong_128b_default( - const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, - const void *XXH_RESTRICT secret, size_t secretLen) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} - (void)seed64; - (void)secret; - (void)secretLen; - return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, - sizeof(XXH3_kSecret), XXH3_accumulate_512, - XXH3_scrambleAcc); +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); } + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM /* - * It's important for performance that XXH3_hashLong is not inlined. + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. */ -XXH_NO_INLINE XXH128_hash_t XXH3_hashLong_128b_withSecret( - const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, - const void *XXH_RESTRICT secret, size_t secretLen) { - - (void)seed64; - return XXH3_hashLong_128b_internal(input, len, (const xxh_u8 *)secret, - secretLen, XXH3_accumulate_512, - XXH3_scrambleAcc); +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + * + * @see @ref streaming_example "Streaming Example" + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; } -XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_withSeed_internal( - const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, - XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble, - XXH3_f_initCustomSecret f_initSec) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} - if (seed64 == 0) - return XXH3_hashLong_128b_internal( - input, len, XXH3_kSecret, sizeof(XXH3_kSecret), f_acc512, f_scramble); - { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} - XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; - f_initSec(secret, seed64); - return XXH3_hashLong_128b_internal(input, len, (const xxh_u8 *)secret, - sizeof(secret), f_acc512, f_scramble); +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} - } +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; } +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif /* - * It's important for performance that XXH3_hashLong is not inlined. + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. */ -XXH_NO_INLINE XXH128_hash_t -XXH3_hashLong_128b_withSeed(const void *input, size_t len, XXH64_hash_t seed64, - const void *XXH_RESTRICT secret, size_t secretLen) { +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } - (void)secret; - (void)secretLen; - return XXH3_hashLong_128b_withSeed_internal( - input, len, seed64, XXH3_accumulate_512, XXH3_scrambleAcc, - XXH3_initCustomSecret); + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ -} + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); -typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void *XXH_RESTRICT, size_t, - XXH64_hash_t, - const void *XXH_RESTRICT, size_t); + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } -XXH_FORCE_INLINE XXH128_hash_t -XXH3_128bits_internal(const void *input, size_t len, XXH64_hash_t seed64, - const void *XXH_RESTRICT secret, size_t secretLen, - XXH3_hashLong128_f f_hl128) { - - XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); - /* - * If an action is to be taken if `secret` conditions are not respected, - * it should be done here. - * For now, it's a contract pre-condition. - * Adding a check and a branch here would cost performance at every hash. - */ - if (len <= 16) - return XXH3_len_0to16_128b((const xxh_u8 *)input, len, - (const xxh_u8 *)secret, seed64); - if (len <= 128) - return XXH3_len_17to128_128b((const xxh_u8 *)input, len, - (const xxh_u8 *)secret, secretLen, seed64); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_128b((const xxh_u8 *)input, len, - (const xxh_u8 *)secret, secretLen, seed64); - return f_hl128(input, len, seed64, secret, secretLen); + return XXH_OK; +} +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); } -/* === Public XXH128 API === */ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void *input, size_t len) { +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; - return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, - sizeof(XXH3_kSecret), - XXH3_hashLong_128b_default); + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); } +#endif /* !XXH_NO_STREAM */ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *input, - size_t len, - const void *secret, - size_t secretSize) { - return XXH3_128bits_internal(input, len, 0, (const xxh_u8 *)secret, - secretSize, XXH3_hashLong_128b_withSecret); +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void *input, - size_t len, - XXH64_hash_t seed) { - - return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, - sizeof(XXH3_kSecret), - XXH3_hashLong_128b_withSeed); - +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH128(const void *input, size_t len, - XXH64_hash_t seed) { +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); - return XXH3_128bits_withSeed(input, len, seed); + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } } -/* === XXH3 128-bit streaming === */ +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} /* - * All the functions are actually the same as for 64-bit streaming variant. - * The only difference is the finalization routine. + * A bit slower than XXH3_mix16B, but handles multiply by zero better. */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t *statePtr) { +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); - if (statePtr == NULL) return XXH_ERROR; - XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret( - XXH3_state_t *statePtr, const void *secret, size_t secretSize) { +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} - if (statePtr == NULL) return XXH_ERROR; - XXH3_reset_internal(statePtr, 0, secret, secretSize); - if (secret == NULL) return XXH_ERROR; - if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; - return XXH_OK; +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t *statePtr, - XXH64_hash_t seed) { +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} - if (statePtr == NULL) return XXH_ERROR; - if (seed == 0) return XXH3_128bits_reset(statePtr); - if (seed != statePtr->seed) - XXH3_initCustomSecret(statePtr->customSecret, seed); - XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t *state, - const void *input, - size_t len) { - return XXH3_update(state, (const xxh_u8 *)input, len, XXH3_accumulate_512, - XXH3_scrambleAcc); +/* === Public XXH128 API === */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); } -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t *state) { - - const unsigned char *const secret = - (state->extSecret == NULL) ? state->customSecret : state->extSecret; - if (state->totalLen > XXH3_MIDSIZE_MAX) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} - XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; - XXH3_digest_long(acc, state, secret); - XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= - sizeof(acc) + XXH_SECRET_MERGEACCS_START); - { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} - XXH128_hash_t h128; - h128.low64 = XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)state->totalLen * XXH_PRIME64_1); - h128.high64 = - XXH3_mergeAccs(acc, - secret + state->secretLimit + XXH_STRIPE_LEN - - sizeof(acc) - XXH_SECRET_MERGEACCS_START, - ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); - return h128; +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} - } +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} - } - /* len <= XXH3_MIDSIZE_MAX : short code */ - if (state->seed) - return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, - state->seed); - return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), - secret, state->secretLimit + XXH_STRIPE_LEN); +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); } +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ - /* 128-bit utility functions */ - - #include /* memcmp, memcpy */ +#include /* memcmp, memcpy */ /* return : 1 is equal, 0 if different */ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { - - /* note : XXH128_hash_t is compact, it has no padding byte */ - return !(memcmp(&h1, &h2, sizeof(h1))); - +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); } /* This prototype is compatible with stdlib's qsort(). - * return : >0 if *h128_1 > *h128_2 - * <0 if *h128_1 < *h128_2 - * =0 if *h128_1 == *h128_2 */ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API int XXH128_cmp(const void *h128_1, const void *h128_2) { + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} - XXH128_hash_t const h1 = *(const XXH128_hash_t *)h128_1; - XXH128_hash_t const h2 = *(const XXH128_hash_t *)h128_2; - int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); - /* note : bets that, in most cases, hash values are different */ - if (hcmp) return hcmp; - return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); } -/*====== Canonical representation ======*/ -/*! @ingroup xxh3_family */ -XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t *dst, - XXH128_hash_t hash) { +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} - XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) { - hash.high64 = XXH_swap64(hash.high64); - hash.low64 = XXH_swap64(hash.low64); - } +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif - memcpy(dst, &hash.high64, sizeof(hash.high64)); - memcpy((char *)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; ndigest + 8); - return h; -} - /* Pop our optimization override from above */ - #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ - && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__OPTIMIZE__) && \ - !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ - #pragma GCC pop_options - #endif +/* Pop our optimization override from above */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC pop_options +#endif - #endif /* XXH_NO_LONG_LONG */ +#endif /* XXH_NO_LONG_LONG */ - #endif /* XXH_NO_XXH3 */ +#endif /* XXH_NO_XXH3 */ /*! * @} */ -#endif /* XXH_IMPLEMENTATION */ - -#if defined(__cplusplus) +#endif /* XXH_IMPLEMENTATION */ -} +#if defined (__cplusplus) +} /* extern "C" */ #endif - diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index ea8f1423..48e32996 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -401,6 +401,10 @@ static void usage(u8 *argv0, int more_help) { SAYF("Compiled with _AFL_DOCUMENT_MUTATIONS.\n"); #endif +#ifdef _AFL_SPECIAL_PERFORMANCE + SAYF("Compiled with special performance options for this specific system, it might not work on other platforms!\n"); +#endif + SAYF("For additional help please consult %s/README.md :)\n\n", doc_path); exit(1); diff --git a/src/afl-performance.c b/src/afl-performance.c index 07c1b527..22cf4dec 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -2,9 +2,18 @@ #include "afl-fuzz.h" #include "types.h" +#ifdef _HAVE_AVX2 +#define T1HA0_AESNI_AVAILABLE 1 +#define T1HA_USE_FAST_ONESHOT_READ 1 +#define T1HA_USE_INDIRECT_FUNCTIONS 1 +#define T1HA_IA32AES_NAME XXH3_64bits +#include "t1ha0_ia32aes_b.h" +#else #define XXH_INLINE_ALL #include "xxhash.h" #undef XXH_INLINE_ALL +#endif + void rand_set_seed(afl_state_t *afl, s64 init_seed) { diff --git a/utils/bench/Makefile b/utils/bench/Makefile new file mode 100644 index 00000000..e7d2f3a1 --- /dev/null +++ b/utils/bench/Makefile @@ -0,0 +1,8 @@ +all: hash + +hash: hash.c + gcc -O3 -mavx2 -march=native -I../../include -o hash hash.c + +clean: + rm -f hash + diff --git a/utils/bench/README.md b/utils/bench/README.md new file mode 100644 index 00000000..772c117b --- /dev/null +++ b/utils/bench/README.md @@ -0,0 +1,2 @@ +# Internal AFL++ benchmarking + diff --git a/utils/bench/hash.c b/utils/bench/hash.c new file mode 100644 index 00000000..013a5321 --- /dev/null +++ b/utils/bench/hash.c @@ -0,0 +1,42 @@ +#include +#include +#include + +#define T1HA0_AESNI_AVAILABLE 1 +#define T1HA_USE_FAST_ONESHOT_READ 1 +#define T1HA_USE_INDIRECT_FUNCTIONS 1 +#define T1HA_IA32AES_NAME t1ha0_ia32aes +#include "t1ha0_ia32aes_b.h" + +#define XXH_INLINE_ALL +#include "xxhash.h" +#undef XXH_INLINE_ALL + +int main() { + char *data = malloc(4097); + struct timespec start, end; + long long duration; + int i; + uint64_t res; + + clock_gettime(CLOCK_MONOTONIC, &start); + for (i = 0; i < 100000000; ++i) { + res = XXH3_64bits(data, 4097); + memcpy(data + 16, (char*)&res, 8); + } + clock_gettime(CLOCK_MONOTONIC, &end); + duration = (end.tv_sec - start.tv_sec) * 1000000000LL + (end.tv_nsec - start.tv_nsec); + printf("xxh3 duration: %lld ns\n", duration); + + memset(data, 0, 4097); + clock_gettime(CLOCK_MONOTONIC, &start); + for (i = 0; i < 100000000; ++i) { + res = t1ha0_ia32aes(data, 4097); + memcpy(data + 16, (char*)&res, 8); + } + clock_gettime(CLOCK_MONOTONIC, &end); + duration = (end.tv_sec - start.tv_sec) * 1000000000LL + (end.tv_nsec - start.tv_nsec); + printf("t1ha0_ia32aes duration: %lld ns\n", duration); + + return 0; +} -- cgit 1.4.1 From 369fce9c85bf3b850a7109e4604fee71f694d2cb Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 8 Feb 2024 15:13:46 +0100 Subject: code format --- TODO.md | 4 +- docs/Changelog.md | 18 + include/cmplog.h | 7 +- include/envs.h | 2 + include/t1ha.h | 527 +-- include/t1ha0_ia32aes_b.h | 116 +- include/t1ha_bits.h | 1466 ++++--- include/t1ha_selfcheck.h | 15 +- include/xxhash.h | 10283 ++++++++++++++++++++++++-------------------- src/afl-fuzz-redqueen.c | 2 + src/afl-fuzz.c | 4 +- src/afl-performance.c | 17 +- utils/bench/hash.c | 31 +- 13 files changed, 6853 insertions(+), 5639 deletions(-) (limited to 'utils') diff --git a/TODO.md b/TODO.md index f2e3963f..d47372b8 100644 --- a/TODO.md +++ b/TODO.md @@ -2,17 +2,15 @@ ## Must - - UI revamp - hardened_usercopy=0 page_alloc.shuffle=0 - add value_profile but only enable after 15 minutes without finds - - cmplog max len, cmplog max items envs? + - cmplog max items env? - adapt MOpt to new mutation engine - Update afl->pending_not_fuzzed for MOpt - cmplog rtn sanity check on fixed length? currently we ignore the length - afl-showmap -f support - afl-fuzz multicore wrapper script - when trimming then perform crash detection - - problem: either -L0 and/or -p mmopt results in zero new coverage ## Should diff --git a/docs/Changelog.md b/docs/Changelog.md index e5169daf..3415150a 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -4,13 +4,31 @@ release of the tool. See README.md for the general instruction manual. ### Version ++4.20a (dev) + ! A new forkserver communication model is now introduced. afl-fuzz is + backward compatible to old compiled targets if they are not built + for CMPLOG/Redqueen, but new compiled targets will not work with + old afl-fuzz versions! + ! Recompiled all targets that are instrumented for CMPLOG/Redqueen! + - AFL++ now supports up to 4 billion coverage edges, up from 6 million. + - New compile option: `make PERFORMANCE=1` - this will enable special + CPU dependent optimizations that make everything more performant - but + the binaries will likely won't work on different platforms. Also + enables a faster hasher if the CPU requirements are met. + - The persistent record feature (see config.h) was expanded to also + support replay, thanks to @quarta-qti ! - afl-fuzz: - the new deterministic fuzzing feature is now activated by default, deactivate with -z. Parameters -d and -D are ignored. + - small improvements to CMPLOG/redqueen + - workround for a bug with MOpt -L when used with -M - in the future + we will either remove or rewrite MOpt. - afl-cc: - added collision free caller instrumentation to LTO mode. activate with `AFL_LLVM_LTO_CALLER=1`. You can set a max depth to go through single block functions with `AFL_LLVM_LTO_CALLER_DEPTH` (default 0) + - Minor edits to afl-persistent-config + - Prevent temporary files being left behind on aborted afl-whatsup + - More CPU benchmarks added to benchmark/ ### Version ++4.10c (release) diff --git a/include/cmplog.h b/include/cmplog.h index 589570fe..a6162b59 100644 --- a/include/cmplog.h +++ b/include/cmplog.h @@ -41,13 +41,12 @@ #define CMP_TYPE_INS 0 #define CMP_TYPE_RTN 1 -struct cmp_header { +struct cmp_header { // 16 bit = 2 bytes unsigned hits : 6; // up to 63 entries, we have CMP_MAP_H = 32 - unsigned shape : 5; // 31+1 bytes - unsigned type : 1; // 4, we use 3: none, rtn, cmp + unsigned shape : 5; // 31+1 bytes max + unsigned type : 1; // 2: cmp, rtn unsigned attribute : 4; // 16 for arithmetic comparison types - //unsigned reserved : 6; } __attribute__((packed)); diff --git a/include/envs.h b/include/envs.h index 8f342553..d32e2f92 100644 --- a/include/envs.h +++ b/include/envs.h @@ -64,6 +64,8 @@ static char *afl_environment_variables[] = { "AFL_REAL_LD", "AFL_LD_PRELOAD", "AFL_LD_VERBOSE", "AFL_LLVM_ALLOWLIST", "AFL_LLVM_DENYLIST", "AFL_LLVM_BLOCKLIST", "AFL_CMPLOG", "AFL_LLVM_CMPLOG", "AFL_GCC_CMPLOG", "AFL_LLVM_INSTRIM", "AFL_LLVM_CALLER", "AFL_LLVM_CTX", + "AFL_LLVM_LTO_CALLER", "AFL_LLVM_LTO_CTX", "AFL_LLVM_LTO_CALLER_DEPTH", + "AFL_LLVM_LTO_CTX_DEPTH", "AFL_LLVM_CALLER_DEPTH", "AFL_LLVM_CTX_DEPTH", "AFL_LLVM_CTX_K", "AFL_LLVM_DICT2FILE", "AFL_LLVM_DICT2FILE_NO_MAIN", "AFL_LLVM_DOCUMENT_IDS", "AFL_LLVM_INSTRIM_LOOPHEAD", "AFL_LLVM_INSTRUMENT", "AFL_LLVM_LTO_AUTODICTIONARY", "AFL_LLVM_AUTODICTIONARY", diff --git a/include/t1ha.h b/include/t1ha.h index 498f0dd6..1af29395 100644 --- a/include/t1ha.h +++ b/include/t1ha.h @@ -172,56 +172,56 @@ #define T1HA_VERSION_RELEASE 1 #ifndef __has_attribute -#define __has_attribute(x) (0) + #define __has_attribute(x) (0) #endif #ifndef __has_include -#define __has_include(x) (0) + #define __has_include(x) (0) #endif #ifndef __GNUC_PREREQ -#if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) -#else -#define __GNUC_PREREQ(maj, min) 0 -#endif -#endif /* __GNUC_PREREQ */ + #if defined(__GNUC__) && defined(__GNUC_MINOR__) + #define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) + #else + #define __GNUC_PREREQ(maj, min) 0 + #endif +#endif /* __GNUC_PREREQ */ #ifndef __CLANG_PREREQ -#ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) -#else -#define __CLANG_PREREQ(maj, min) (0) -#endif -#endif /* __CLANG_PREREQ */ + #ifdef __clang__ + #define __CLANG_PREREQ(maj, min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) + #else + #define __CLANG_PREREQ(maj, min) (0) + #endif +#endif /* __CLANG_PREREQ */ #ifndef __LCC_PREREQ -#ifdef __LCC__ -#define __LCC_PREREQ(maj, min) \ - ((__LCC__ << 16) + __LCC_MINOR__ >= ((maj) << 16) + (min)) -#else -#define __LCC_PREREQ(maj, min) (0) -#endif -#endif /* __LCC_PREREQ */ + #ifdef __LCC__ + #define __LCC_PREREQ(maj, min) \ + ((__LCC__ << 16) + __LCC_MINOR__ >= ((maj) << 16) + (min)) + #else + #define __LCC_PREREQ(maj, min) (0) + #endif +#endif /* __LCC_PREREQ */ /*****************************************************************************/ #ifdef _MSC_VER -/* Avoid '16' bytes padding added after data member 't1ha_context::total' - * and other warnings from std-headers if warning-level > 3. */ -#pragma warning(push, 3) + /* Avoid '16' bytes padding added after data member 't1ha_context::total' + * and other warnings from std-headers if warning-level > 3. */ + #pragma warning(push, 3) #endif #if defined(__cplusplus) && __cplusplus >= 201103L -#include -#include -#include + #include + #include + #include #else -#include -#include -#include + #include + #include + #include #endif /*****************************************************************************/ @@ -234,18 +234,18 @@ defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) -#ifndef __ia32__ -/* LY: define neutral __ia32__ for x86 and x86-64 archs */ -#define __ia32__ 1 -#endif /* __ia32__ */ -#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64) || defined(_M_X64)) -/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ -#define __amd64__ 1 -#endif /* __amd64__ */ -#endif /* all x86 */ - -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + #ifndef __ia32__ + /* LY: define neutral __ia32__ for x86 and x86-64 archs */ + #define __ia32__ 1 + #endif /* __ia32__ */ + #if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) + /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ + #define __amd64__ 1 + #endif /* __amd64__ */ +#endif /* all x86 */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ !defined(__ORDER_BIG_ENDIAN__) /* *INDENT-OFF* */ @@ -267,160 +267,168 @@ defined(__NETBSD__) || defined(__NetBSD__) || \ defined(HAVE_SYS_PARAM_H) || __has_include() #include -#endif /* OS */ +#endif /* OS */ /* *INDENT-ON* */ /* clang-format on */ -#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) -#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN -#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN -#define __BYTE_ORDER__ __BYTE_ORDER -#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) -#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN -#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN -#define __BYTE_ORDER__ _BYTE_ORDER -#else -#define __ORDER_LITTLE_ENDIAN__ 1234 -#define __ORDER_BIG_ENDIAN__ 4321 - -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) -#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ - -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) -#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ - -#else -#error __BYTE_ORDER__ should be defined. -#endif /* Arch */ - -#endif + #if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) + #define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN + #define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN + #define __BYTE_ORDER__ __BYTE_ORDER + #elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) + #define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN + #define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN + #define __BYTE_ORDER__ _BYTE_ORDER + #else + #define __ORDER_LITTLE_ENDIAN__ 1234 + #define __ORDER_BIG_ENDIAN__ 4321 + + #if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || \ + defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || \ + defined(__MIPSEL) || defined(_M_ARM) || defined(_M_ARM64) || \ + defined(__e2k__) || defined(__elbrus_4c__) || \ + defined(__elbrus_8c__) || defined(__bfin__) || defined(__BFIN__) || \ + defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || \ + defined(__ia32__) || defined(__CYGWIN__) || defined(_WIN64) || \ + defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) + #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + + #elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || \ + defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || \ + defined(__MIPSEB) || defined(__m68k__) || defined(M68000) || \ + defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || \ + defined(__THW_370__) || defined(__s390__) || defined(__s390x__) || \ + defined(__SYSC_ZARCH__) + #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + + #else + #error __BYTE_ORDER__ should be defined. + #endif /* Arch */ + + #endif #endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ /*****************************************************************************/ #ifndef __dll_export -#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) -#if defined(__GNUC__) || __has_attribute(dllexport) -#define __dll_export __attribute__((dllexport)) -#else -#define __dll_export __declspec(dllexport) -#endif -#elif defined(__GNUC__) || __has_attribute(__visibility__) -#define __dll_export __attribute__((__visibility__("default"))) -#else -#define __dll_export -#endif -#endif /* __dll_export */ + #if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) + #if defined(__GNUC__) || __has_attribute(dllexport) + #define __dll_export __attribute__((dllexport)) + #else + #define __dll_export __declspec(dllexport) + #endif + #elif defined(__GNUC__) || __has_attribute(__visibility__) + #define __dll_export __attribute__((__visibility__("default"))) + #else + #define __dll_export + #endif +#endif /* __dll_export */ #ifndef __dll_import -#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) -#if defined(__GNUC__) || __has_attribute(dllimport) -#define __dll_import __attribute__((dllimport)) -#else -#define __dll_import __declspec(dllimport) -#endif -#elif defined(__GNUC__) || __has_attribute(__visibility__) -#define __dll_import __attribute__((__visibility__("default"))) -#else -#define __dll_import -#endif -#endif /* __dll_import */ + #if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) + #if defined(__GNUC__) || __has_attribute(dllimport) + #define __dll_import __attribute__((dllimport)) + #else + #define __dll_import __declspec(dllimport) + #endif + #elif defined(__GNUC__) || __has_attribute(__visibility__) + #define __dll_import __attribute__((__visibility__("default"))) + #else + #define __dll_import + #endif +#endif /* __dll_import */ #ifndef __force_inline -#ifdef _MSC_VER -#define __force_inline __forceinline -#elif __GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__) -#define __force_inline __inline __attribute__((__always_inline__)) -#else -#define __force_inline __inline -#endif -#endif /* __force_inline */ + #ifdef _MSC_VER + #define __force_inline __forceinline + #elif __GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__) + #define __force_inline __inline __attribute__((__always_inline__)) + #else + #define __force_inline __inline + #endif +#endif /* __force_inline */ #ifndef T1HA_API -#if defined(t1ha_EXPORTS) -#define T1HA_API __dll_export -#elif defined(t1ha_IMPORTS) -#define T1HA_API __dll_import -#else -#define T1HA_API -#endif -#endif /* T1HA_API */ + #if defined(t1ha_EXPORTS) + #define T1HA_API __dll_export + #elif defined(t1ha_IMPORTS) + #define T1HA_API __dll_import + #else + #define T1HA_API + #endif +#endif /* T1HA_API */ #if defined(_MSC_VER) && defined(__ia32__) -#define T1HA_ALIGN_PREFIX __declspec(align(32)) /* required only for SIMD */ + #define T1HA_ALIGN_PREFIX __declspec(align(32)) /* required only for SIMD */ #else -#define T1HA_ALIGN_PREFIX -#endif /* _MSC_VER */ + #define T1HA_ALIGN_PREFIX +#endif /* _MSC_VER */ #if defined(__GNUC__) && defined(__ia32__) -#define T1HA_ALIGN_SUFFIX \ - __attribute__((__aligned__(32))) /* required only for SIMD */ + #define T1HA_ALIGN_SUFFIX \ + __attribute__((__aligned__(32))) /* required only for SIMD */ #else -#define T1HA_ALIGN_SUFFIX -#endif /* GCC x86 */ + #define T1HA_ALIGN_SUFFIX +#endif /* GCC x86 */ #ifndef T1HA_USE_INDIRECT_FUNCTIONS -/* GNU ELF indirect functions usage control. For more info please see - * https://en.wikipedia.org/wiki/Executable_and_Linkable_Format - * and https://sourceware.org/glibc/wiki/GNU_IFUNC */ -#if defined(__ELF__) && defined(__amd64__) && \ - (__has_attribute(__ifunc__) || \ - (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && \ - !defined(__SANITIZE_ADDRESS__) && !defined(__SSP_ALL__))) -/* Enable gnu_indirect_function by default if : - * - ELF AND x86_64 - * - attribute(__ifunc__) is available OR - * GCC >= 4 WITHOUT -fsanitize=address NOR -fstack-protector-all */ -#define T1HA_USE_INDIRECT_FUNCTIONS 1 -#else -#define T1HA_USE_INDIRECT_FUNCTIONS 0 -#endif -#endif /* T1HA_USE_INDIRECT_FUNCTIONS */ + /* GNU ELF indirect functions usage control. For more info please see + * https://en.wikipedia.org/wiki/Executable_and_Linkable_Format + * and https://sourceware.org/glibc/wiki/GNU_IFUNC */ + #if defined(__ELF__) && defined(__amd64__) && \ + (__has_attribute(__ifunc__) || \ + (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && \ + !defined(__SANITIZE_ADDRESS__) && !defined(__SSP_ALL__))) + /* Enable gnu_indirect_function by default if : + * - ELF AND x86_64 + * - attribute(__ifunc__) is available OR + * GCC >= 4 WITHOUT -fsanitize=address NOR -fstack-protector-all */ + #define T1HA_USE_INDIRECT_FUNCTIONS 1 + #else + #define T1HA_USE_INDIRECT_FUNCTIONS 0 + #endif +#endif /* T1HA_USE_INDIRECT_FUNCTIONS */ #if __GNUC_PREREQ(4, 0) -#pragma GCC visibility push(hidden) -#endif /* __GNUC_PREREQ(4,0) */ + #pragma GCC visibility push(hidden) +#endif /* __GNUC_PREREQ(4,0) */ #ifdef __cplusplus extern "C" { + #endif typedef union T1HA_ALIGN_PREFIX t1ha_state256 { - uint8_t bytes[32]; + + uint8_t bytes[32]; uint32_t u32[8]; uint64_t u64[4]; struct { + uint64_t a, b, c, d; + } n; + } t1ha_state256_t T1HA_ALIGN_SUFFIX; typedef struct t1ha_context { + t1ha_state256_t state; t1ha_state256_t buffer; - size_t partial; - uint64_t total; + size_t partial; + uint64_t total; + } t1ha_context_t; #ifdef _MSC_VER -#pragma warning(pop) + #pragma warning(pop) #endif /****************************************************************************** @@ -443,37 +451,37 @@ T1HA_API int t1ha_selfcheck__t1ha2_atonce(void); T1HA_API int t1ha_selfcheck__t1ha2_atonce128(void); T1HA_API int t1ha_selfcheck__t1ha2_stream(void); T1HA_API int t1ha_selfcheck__t1ha2(void); -#endif /* T1HA2_DISABLED */ +#endif /* T1HA2_DISABLED */ #ifndef T1HA1_DISABLED T1HA_API int t1ha_selfcheck__t1ha1_le(void); T1HA_API int t1ha_selfcheck__t1ha1_be(void); T1HA_API int t1ha_selfcheck__t1ha1(void); -#endif /* T1HA1_DISABLED */ +#endif /* T1HA1_DISABLED */ #ifndef T1HA0_DISABLED T1HA_API int t1ha_selfcheck__t1ha0_32le(void); T1HA_API int t1ha_selfcheck__t1ha0_32be(void); T1HA_API int t1ha_selfcheck__t1ha0(void); -/* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */ -#ifndef T1HA0_AESNI_AVAILABLE -#if defined(__e2k__) || \ - (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800)) -#define T1HA0_AESNI_AVAILABLE 1 -#else -#define T1HA0_AESNI_AVAILABLE 0 -#endif -#endif /* ifndef T1HA0_AESNI_AVAILABLE */ - -#if T1HA0_AESNI_AVAILABLE + /* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */ + #ifndef T1HA0_AESNI_AVAILABLE + #if defined(__e2k__) || \ + (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800)) + #define T1HA0_AESNI_AVAILABLE 1 + #else + #define T1HA0_AESNI_AVAILABLE 0 + #endif + #endif /* ifndef T1HA0_AESNI_AVAILABLE */ + + #if T1HA0_AESNI_AVAILABLE T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_noavx(void); T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx(void); -#ifndef __e2k__ + #ifndef __e2k__ T1HA_API int t1ha_selfcheck__t1ha0_ia32aes_avx2(void); -#endif -#endif /* if T1HA0_AESNI_AVAILABLE */ -#endif /* T1HA0_DISABLED */ + #endif + #endif /* if T1HA0_AESNI_AVAILABLE */ +#endif /* T1HA0_DISABLED */ /****************************************************************************** * @@ -521,7 +529,7 @@ T1HA_API void t1ha2_update(t1ha_context_t *__restrict ctx, T1HA_API uint64_t t1ha2_final(t1ha_context_t *__restrict ctx, uint64_t *__restrict extra_result /* optional */); -#endif /* T1HA2_DISABLED */ +#endif /* T1HA2_DISABLED */ /****************************************************************************** * @@ -546,7 +554,7 @@ T1HA_API uint64_t t1ha1_le(const void *data, size_t length, uint64_t seed); /* The big-endian variant. */ T1HA_API uint64_t t1ha1_be(const void *data, size_t length, uint64_t seed); -#endif /* T1HA1_DISABLED */ +#endif /* T1HA1_DISABLED */ /****************************************************************************** * @@ -589,131 +597,142 @@ uint64_t t1ha0_32le(const void *data, size_t length, uint64_t seed); /* The big-endian variant for 32-bit CPU. */ uint64_t t1ha0_32be(const void *data, size_t length, uint64_t seed); -/* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */ -#ifndef T1HA0_AESNI_AVAILABLE -#if defined(__e2k__) || \ - (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800)) -#define T1HA0_AESNI_AVAILABLE 1 -#else -#define T1HA0_AESNI_AVAILABLE 0 -#endif -#endif /* T1HA0_AESNI_AVAILABLE */ - -/* Define T1HA0_RUNTIME_SELECT to 0 for disable dispatching t1ha0 at runtime. */ -#ifndef T1HA0_RUNTIME_SELECT -#if T1HA0_AESNI_AVAILABLE && !defined(__e2k__) -#define T1HA0_RUNTIME_SELECT 1 -#else -#define T1HA0_RUNTIME_SELECT 0 -#endif -#endif /* T1HA0_RUNTIME_SELECT */ - -#if !T1HA0_RUNTIME_SELECT && !defined(T1HA0_USE_DEFINE) -#if defined(__LCC__) -#define T1HA0_USE_DEFINE 1 -#else -#define T1HA0_USE_DEFINE 0 -#endif -#endif /* T1HA0_USE_DEFINE */ - -#if T1HA0_AESNI_AVAILABLE + /* Define T1HA0_AESNI_AVAILABLE to 0 for disable AES-NI support. */ + #ifndef T1HA0_AESNI_AVAILABLE + #if defined(__e2k__) || \ + (defined(__ia32__) && (!defined(_M_IX86) || _MSC_VER > 1800)) + #define T1HA0_AESNI_AVAILABLE 1 + #else + #define T1HA0_AESNI_AVAILABLE 0 + #endif + #endif /* T1HA0_AESNI_AVAILABLE */ + + /* Define T1HA0_RUNTIME_SELECT to 0 for disable dispatching t1ha0 at runtime. + */ + #ifndef T1HA0_RUNTIME_SELECT + #if T1HA0_AESNI_AVAILABLE && !defined(__e2k__) + #define T1HA0_RUNTIME_SELECT 1 + #else + #define T1HA0_RUNTIME_SELECT 0 + #endif + #endif /* T1HA0_RUNTIME_SELECT */ + + #if !T1HA0_RUNTIME_SELECT && !defined(T1HA0_USE_DEFINE) + #if defined(__LCC__) + #define T1HA0_USE_DEFINE 1 + #else + #define T1HA0_USE_DEFINE 0 + #endif + #endif /* T1HA0_USE_DEFINE */ + + #if T1HA0_AESNI_AVAILABLE uint64_t t1ha0_ia32aes_noavx(const void *data, size_t length, uint64_t seed); uint64_t t1ha0_ia32aes_avx(const void *data, size_t length, uint64_t seed); -#ifndef __e2k__ + #ifndef __e2k__ uint64_t t1ha0_ia32aes_avx2(const void *data, size_t length, uint64_t seed); -#endif -#endif /* T1HA0_AESNI_AVAILABLE */ + #endif + #endif /* T1HA0_AESNI_AVAILABLE */ -#if T1HA0_RUNTIME_SELECT + #if T1HA0_RUNTIME_SELECT typedef uint64_t (*t1ha0_function_t)(const void *, size_t, uint64_t); T1HA_API t1ha0_function_t t1ha0_resolve(void); -#if T1HA_USE_INDIRECT_FUNCTIONS + #if T1HA_USE_INDIRECT_FUNCTIONS T1HA_API uint64_t t1ha0(const void *data, size_t length, uint64_t seed); -#else + #else /* Otherwise function pointer will be used. * Unfortunately this may cause some overhead calling. */ T1HA_API extern uint64_t (*t1ha0_funcptr)(const void *data, size_t length, uint64_t seed); static __force_inline uint64_t t1ha0(const void *data, size_t length, uint64_t seed) { + return t1ha0_funcptr(data, length, seed); + } -#endif /* T1HA_USE_INDIRECT_FUNCTIONS */ -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #endif /* T1HA_USE_INDIRECT_FUNCTIONS */ -#if T1HA0_USE_DEFINE + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ - (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) -#if defined(T1HA1_DISABLED) -#define t1ha0 t1ha2_atonce -#else -#define t1ha0 t1ha1_be -#endif /* T1HA1_DISABLED */ -#else /* 32/64 */ -#define t1ha0 t1ha0_32be -#endif /* 32/64 */ + #if T1HA0_USE_DEFINE + + #if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) + #if defined(T1HA1_DISABLED) + #define t1ha0 t1ha2_atonce + #else + #define t1ha0 t1ha1_be + #endif /* T1HA1_DISABLED */ + #else /* 32/64 */ + #define t1ha0 t1ha0_32be + #endif /* 32/64 */ -#else /* T1HA0_USE_DEFINE */ + #else /* T1HA0_USE_DEFINE */ static __force_inline uint64_t t1ha0(const void *data, size_t length, uint64_t seed) { -#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ - (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) -#if defined(T1HA1_DISABLED) + + #if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) + #if defined(T1HA1_DISABLED) return t1ha2_atonce(data, length, seed); -#else + #else return t1ha1_be(data, length, seed); -#endif /* T1HA1_DISABLED */ -#else /* 32/64 */ + #endif /* T1HA1_DISABLED */ + #else /* 32/64 */ return t1ha0_32be(data, length, seed); -#endif /* 32/64 */ + #endif /* 32/64 */ + } -#endif /* !T1HA0_USE_DEFINE */ + #endif /* !T1HA0_USE_DEFINE */ -#else /* !T1HA0_RUNTIME_SELECT && __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ + #else /* !T1HA0_RUNTIME_SELECT && __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ -#if T1HA0_USE_DEFINE + #if T1HA0_USE_DEFINE -#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ - (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) -#if defined(T1HA1_DISABLED) -#define t1ha0 t1ha2_atonce -#else -#define t1ha0 t1ha1_le -#endif /* T1HA1_DISABLED */ -#else /* 32/64 */ -#define t1ha0 t1ha0_32le -#endif /* 32/64 */ + #if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) + #if defined(T1HA1_DISABLED) + #define t1ha0 t1ha2_atonce + #else + #define t1ha0 t1ha1_le + #endif /* T1HA1_DISABLED */ + #else /* 32/64 */ + #define t1ha0 t1ha0_32le + #endif /* 32/64 */ -#else + #else static __force_inline uint64_t t1ha0(const void *data, size_t length, uint64_t seed) { -#if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ - (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) -#if defined(T1HA1_DISABLED) + + #if (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) && \ + (!defined(T1HA1_DISABLED) || !defined(T1HA2_DISABLED)) + #if defined(T1HA1_DISABLED) return t1ha2_atonce(data, length, seed); -#else + #else return t1ha1_le(data, length, seed); -#endif /* T1HA1_DISABLED */ -#else /* 32/64 */ + #endif /* T1HA1_DISABLED */ + #else /* 32/64 */ return t1ha0_32le(data, length, seed); -#endif /* 32/64 */ + #endif /* 32/64 */ + } -#endif /* !T1HA0_USE_DEFINE */ + #endif /* !T1HA0_USE_DEFINE */ -#endif /* !T1HA0_RUNTIME_SELECT */ + #endif /* !T1HA0_RUNTIME_SELECT */ -#endif /* T1HA0_DISABLED */ +#endif /* T1HA0_DISABLED */ #ifdef __cplusplus + } + #endif #if __GNUC_PREREQ(4, 0) -#pragma GCC visibility pop -#endif /* __GNUC_PREREQ(4,0) */ + #pragma GCC visibility pop +#endif /* __GNUC_PREREQ(4,0) */ + diff --git a/include/t1ha0_ia32aes_b.h b/include/t1ha0_ia32aes_b.h index e8e52638..93b16771 100644 --- a/include/t1ha0_ia32aes_b.h +++ b/include/t1ha0_ia32aes_b.h @@ -47,27 +47,34 @@ #if T1HA0_AESNI_AVAILABLE uint64_t T1HA_IA32AES_NAME(const void *data, uint32_t len) { + uint64_t a = 0; uint64_t b = len; if (likely(len > 32)) { + __m128i x = _mm_set_epi64x(a, b); __m128i y = _mm_aesenc_si128(x, _mm_set_epi64x(prime_0, prime_1)); - const __m128i *v = (const __m128i *)data; + const __m128i *v = (const __m128i *)data; const __m128i *const detent = (const __m128i *)((const uint8_t *)data + (len & ~15ul)); data = detent; if (len & 16) { + x = _mm_add_epi64(x, _mm_loadu_si128(v++)); y = _mm_aesenc_si128(x, y); + } + len &= 15; if (v + 7 < detent) { + __m128i salt = y; do { + __m128i t = _mm_aesenc_si128(_mm_loadu_si128(v++), salt); t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); t = _mm_aesdec_si128(t, _mm_loadu_si128(v++)); @@ -82,86 +89,95 @@ uint64_t T1HA_IA32AES_NAME(const void *data, uint32_t len) { t = _mm_aesenc_si128(x, t); x = _mm_add_epi64(y, x); y = t; + } while (v + 7 < detent); + } while (v < detent) { + __m128i v0y = _mm_add_epi64(y, _mm_loadu_si128(v++)); __m128i v1x = _mm_sub_epi64(x, _mm_loadu_si128(v++)); x = _mm_aesdec_si128(x, v0y); y = _mm_aesdec_si128(y, v1x); + } x = _mm_add_epi64(_mm_aesdec_si128(x, _mm_aesenc_si128(y, x)), y); -#if defined(__x86_64__) || defined(_M_X64) -#if defined(__SSE4_1__) || defined(__AVX__) + #if defined(__x86_64__) || defined(_M_X64) + #if defined(__SSE4_1__) || defined(__AVX__) a = _mm_extract_epi64(x, 0); b = _mm_extract_epi64(x, 1); -#else + #else a = _mm_cvtsi128_si64(x); b = _mm_cvtsi128_si64(_mm_unpackhi_epi64(x, x)); -#endif -#else -#if defined(__SSE4_1__) || defined(__AVX__) + #endif + #else + #if defined(__SSE4_1__) || defined(__AVX__) a = (uint32_t)_mm_extract_epi32(x, 0) | (uint64_t)_mm_extract_epi32(x, 1) << 32; b = (uint32_t)_mm_extract_epi32(x, 2) | (uint64_t)_mm_extract_epi32(x, 3) << 32; -#else + #else a = (uint32_t)_mm_cvtsi128_si32(x); a |= (uint64_t)_mm_cvtsi128_si32(_mm_shuffle_epi32(x, 1)) << 32; x = _mm_unpackhi_epi64(x, x); b = (uint32_t)_mm_cvtsi128_si32(x); b |= (uint64_t)_mm_cvtsi128_si32(_mm_shuffle_epi32(x, 1)) << 32; -#endif -#endif -#ifdef __AVX__ + #endif + #endif + #ifdef __AVX__ _mm256_zeroupper(); -#elif !(defined(_X86_64_) || defined(__x86_64__) || defined(_M_X64) || \ - defined(__e2k__)) + #elif !(defined(_X86_64_) || defined(__x86_64__) || defined(_M_X64) || \ + defined(__e2k__)) _mm_empty(); -#endif + #endif + } const uint64_t *v = (const uint64_t *)data; switch (len) { - default: - mixup64(&a, &b, fetch64_le_unaligned(v++), prime_4); - /* fall through */ - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - mixup64(&b, &a, fetch64_le_unaligned(v++), prime_3); - /* fall through */ - case 16: - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - mixup64(&a, &b, fetch64_le_unaligned(v++), prime_2); - /* fall through */ - case 8: - case 7: - case 6: - case 5: - case 4: - case 3: - case 2: - case 1: - mixup64(&b, &a, tail64_le_unaligned(v, len), prime_1); - /* fall through */ - case 0: - return final64(a, b); + + default: + mixup64(&a, &b, fetch64_le_unaligned(v++), prime_4); + /* fall through */ + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + mixup64(&b, &a, fetch64_le_unaligned(v++), prime_3); + /* fall through */ + case 16: + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + mixup64(&a, &b, fetch64_le_unaligned(v++), prime_2); + /* fall through */ + case 8: + case 7: + case 6: + case 5: + case 4: + case 3: + case 2: + case 1: + mixup64(&b, &a, tail64_le_unaligned(v, len), prime_1); + /* fall through */ + case 0: + return final64(a, b); + } + } -#endif /* T1HA0_AESNI_AVAILABLE */ +#endif /* T1HA0_AESNI_AVAILABLE */ #undef T1HA_IA32AES_NAME + diff --git a/include/t1ha_bits.h b/include/t1ha_bits.h index 539369aa..e7a8d53c 100644 --- a/include/t1ha_bits.h +++ b/include/t1ha_bits.h @@ -44,30 +44,30 @@ #pragma once #if defined(_MSC_VER) -#pragma warning(disable : 4201) /* nameless struct/union */ -#if _MSC_VER > 1800 -#pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif /* 1800 */ -#endif /* MSVC */ + #pragma warning(disable : 4201) /* nameless struct/union */ + #if _MSC_VER > 1800 + #pragma warning(disable : 4464) /* relative include path contains '..' */ + #endif /* 1800 */ +#endif /* MSVC */ #include "t1ha.h" #ifndef T1HA_USE_FAST_ONESHOT_READ -/* Define it to 1 for little bit faster code. - * Unfortunately this may triggering a false-positive alarms from Valgrind, - * AddressSanitizer and other similar tool. - * So, define it to 0 for calmness if doubt. */ -#define T1HA_USE_FAST_ONESHOT_READ 1 -#endif /* T1HA_USE_FAST_ONESHOT_READ */ + /* Define it to 1 for little bit faster code. + * Unfortunately this may triggering a false-positive alarms from Valgrind, + * AddressSanitizer and other similar tool. + * So, define it to 0 for calmness if doubt. */ + #define T1HA_USE_FAST_ONESHOT_READ 1 +#endif /* T1HA_USE_FAST_ONESHOT_READ */ /*****************************************************************************/ -#include /* for assert() */ -#include /* for bool */ -#include /* for memcpy() */ +#include /* for assert() */ +#include /* for bool */ +#include /* for memcpy() */ -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ -#error Unsupported byte order. + #error Unsupported byte order. #endif #define T1HA_UNALIGNED_ACCESS__UNABLE 0 @@ -75,534 +75,600 @@ #define T1HA_UNALIGNED_ACCESS__EFFICIENT 2 #ifndef T1HA_SYS_UNALIGNED_ACCESS -#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT -#elif defined(__ia32__) -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT -#elif defined(__e2k__) -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__SLOW -#elif defined(__ARM_FEATURE_UNALIGNED) -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT -#else -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__UNABLE -#endif -#endif /* T1HA_SYS_UNALIGNED_ACCESS */ + #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT + #elif defined(__ia32__) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT + #elif defined(__e2k__) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__SLOW + #elif defined(__ARM_FEATURE_UNALIGNED) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT + #else + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__UNABLE + #endif +#endif /* T1HA_SYS_UNALIGNED_ACCESS */ #define ALIGNMENT_16 2 #define ALIGNMENT_32 4 #if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul -#define ALIGNMENT_64 8 + #define ALIGNMENT_64 8 #else -#define ALIGNMENT_64 4 + #define ALIGNMENT_64 4 #endif #ifndef PAGESIZE -#define PAGESIZE 4096 -#endif /* PAGESIZE */ + #define PAGESIZE 4096 +#endif /* PAGESIZE */ /***************************************************************************/ #ifndef __has_builtin -#define __has_builtin(x) (0) + #define __has_builtin(x) (0) #endif #ifndef __has_warning -#define __has_warning(x) (0) + #define __has_warning(x) (0) #endif #ifndef __has_feature -#define __has_feature(x) (0) + #define __has_feature(x) (0) #endif #ifndef __has_extension -#define __has_extension(x) (0) + #define __has_extension(x) (0) #endif #if __has_feature(address_sanitizer) -#define __SANITIZE_ADDRESS__ 1 + #define __SANITIZE_ADDRESS__ 1 #endif #ifndef __optimize -#if defined(__clang__) && !__has_attribute(__optimize__) -#define __optimize(ops) -#elif defined(__GNUC__) || __has_attribute(__optimize__) -#define __optimize(ops) __attribute__((__optimize__(ops))) -#else -#define __optimize(ops) -#endif -#endif /* __optimize */ + #if defined(__clang__) && !__has_attribute(__optimize__) + #define __optimize(ops) + #elif defined(__GNUC__) || __has_attribute(__optimize__) + #define __optimize(ops) __attribute__((__optimize__(ops))) + #else + #define __optimize(ops) + #endif +#endif /* __optimize */ #ifndef __cold -#if defined(__OPTIMIZE__) -#if defined(__e2k__) -#define __cold __optimize(1) __attribute__((__cold__)) -#elif defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) -/* just put infrequently used functions in separate section */ -#define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") -#elif defined(__GNUC__) || __has_attribute(__cold__) -#define __cold __attribute__((__cold__)) __optimize("Os") -#else -#define __cold __optimize("Os") -#endif -#else -#define __cold -#endif -#endif /* __cold */ + #if defined(__OPTIMIZE__) + #if defined(__e2k__) + #define __cold __optimize(1) __attribute__((__cold__)) + #elif defined(__clang__) && !__has_attribute(__cold__) && \ + __has_attribute(__section__) + /* just put infrequently used functions in separate section */ + #define __cold \ + __attribute__((__section__("text.unlikely"))) __optimize("Os") + #elif defined(__GNUC__) || __has_attribute(__cold__) + #define __cold __attribute__((__cold__)) __optimize("Os") + #else + #define __cold __optimize("Os") + #endif + #else + #define __cold + #endif +#endif /* __cold */ #if __GNUC_PREREQ(4, 4) || defined(__clang__) -#if defined(__ia32__) || defined(__e2k__) -#include -#endif + #if defined(__ia32__) || defined(__e2k__) + #include + #endif -#if defined(__ia32__) && !defined(__cpuid_count) -#include -#endif + #if defined(__ia32__) && !defined(__cpuid_count) + #include + #endif -#if defined(__e2k__) -#include -#endif + #if defined(__e2k__) + #include + #endif -#ifndef likely -#define likely(cond) __builtin_expect(!!(cond), 1) -#endif + #ifndef likely + #define likely(cond) __builtin_expect(!!(cond), 1) + #endif -#ifndef unlikely -#define unlikely(cond) __builtin_expect(!!(cond), 0) -#endif + #ifndef unlikely + #define unlikely(cond) __builtin_expect(!!(cond), 0) + #endif -#if __GNUC_PREREQ(4, 5) || __has_builtin(__builtin_unreachable) -#define unreachable() __builtin_unreachable() -#endif + #if __GNUC_PREREQ(4, 5) || __has_builtin(__builtin_unreachable) + #define unreachable() __builtin_unreachable() + #endif -#define bswap64(v) __builtin_bswap64(v) -#define bswap32(v) __builtin_bswap32(v) -#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16) -#define bswap16(v) __builtin_bswap16(v) -#endif + #define bswap64(v) __builtin_bswap64(v) + #define bswap32(v) __builtin_bswap32(v) + #if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16) + #define bswap16(v) __builtin_bswap16(v) + #endif -#if !defined(__maybe_unused) && \ - (__GNUC_PREREQ(4, 3) || __has_attribute(__unused__)) -#define __maybe_unused __attribute__((__unused__)) -#endif + #if !defined(__maybe_unused) && \ + (__GNUC_PREREQ(4, 3) || __has_attribute(__unused__)) + #define __maybe_unused __attribute__((__unused__)) + #endif -#if !defined(__always_inline) && \ - (__GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__)) -#define __always_inline __inline __attribute__((__always_inline__)) -#endif + #if !defined(__always_inline) && \ + (__GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__)) + #define __always_inline __inline __attribute__((__always_inline__)) + #endif -#if defined(__e2k__) + #if defined(__e2k__) -#if __iset__ >= 3 -#define mul_64x64_high(a, b) __builtin_e2k_umulhd(a, b) -#endif /* __iset__ >= 3 */ + #if __iset__ >= 3 + #define mul_64x64_high(a, b) __builtin_e2k_umulhd(a, b) + #endif /* __iset__ >= 3 */ + + #if __iset__ >= 5 +static __maybe_unused __always_inline unsigned e2k_add64carry_first( + uint64_t base, uint64_t addend, uint64_t *sum) { -#if __iset__ >= 5 -static __maybe_unused __always_inline unsigned -e2k_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) { *sum = base + addend; return (unsigned)__builtin_e2k_addcd_c(base, addend, 0); + } -#define add64carry_first(base, addend, sum) \ - e2k_add64carry_first(base, addend, sum) +\ + #define add64carry_first(base, addend, sum) \ + e2k_add64carry_first(base, addend, sum) + +static __maybe_unused __always_inline unsigned e2k_add64carry_next( + unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) { -static __maybe_unused __always_inline unsigned -e2k_add64carry_next(unsigned carry, uint64_t base, uint64_t addend, - uint64_t *sum) { *sum = __builtin_e2k_addcd(base, addend, carry); return (unsigned)__builtin_e2k_addcd_c(base, addend, carry); + } -#define add64carry_next(carry, base, addend, sum) \ - e2k_add64carry_next(carry, base, addend, sum) +\ + #define add64carry_next(carry, base, addend, sum) \ + e2k_add64carry_next(carry, base, addend, sum) -static __maybe_unused __always_inline void e2k_add64carry_last(unsigned carry, - uint64_t base, - uint64_t addend, +static __maybe_unused __always_inline void e2k_add64carry_last(unsigned carry, + uint64_t base, + uint64_t addend, uint64_t *sum) { + *sum = __builtin_e2k_addcd(base, addend, carry); + } -#define add64carry_last(carry, base, addend, sum) \ - e2k_add64carry_last(carry, base, addend, sum) -#endif /* __iset__ >= 5 */ +\ + #define add64carry_last(carry, base, addend, sum) \ + e2k_add64carry_last(carry, base, addend, sum) + #endif /* __iset__ >= 5 */ -#define fetch64_be_aligned(ptr) ((uint64_t)__builtin_e2k_ld_64s_be(ptr)) -#define fetch32_be_aligned(ptr) ((uint32_t)__builtin_e2k_ld_32u_be(ptr)) + #define fetch64_be_aligned(ptr) ((uint64_t)__builtin_e2k_ld_64s_be(ptr)) + #define fetch32_be_aligned(ptr) ((uint32_t)__builtin_e2k_ld_32u_be(ptr)) -#endif /* __e2k__ Elbrus */ + #endif /* __e2k__ Elbrus */ #elif defined(_MSC_VER) -#if _MSC_FULL_VER < 190024234 && defined(_M_IX86) -#pragma message( \ - "For AES-NI at least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required.") -#endif -#if _MSC_FULL_VER < 191526730 -#pragma message( \ - "It is recommended to use \"Microsoft C/C++ Compiler\" version 19.15.26730 (Visual Studio 2017 15.8) or newer.") -#endif -#if _MSC_FULL_VER < 180040629 -#error At least "Microsoft C/C++ Compiler" version 18.00.40629 (Visual Studio 2013 Update 5) is required. -#endif - -#pragma warning(push, 1) - -#include -#include -#define likely(cond) (cond) -#define unlikely(cond) (cond) -#define unreachable() __assume(0) -#define bswap64(v) _byteswap_uint64(v) -#define bswap32(v) _byteswap_ulong(v) -#define bswap16(v) _byteswap_ushort(v) -#define rot64(v, s) _rotr64(v, s) -#define rot32(v, s) _rotr(v, s) -#define __always_inline __forceinline - -#if defined(_M_X64) || defined(_M_IA64) -#pragma intrinsic(_umul128) -#define mul_64x64_128(a, b, ph) _umul128(a, b, ph) -#pragma intrinsic(_addcarry_u64) -#define add64carry_first(base, addend, sum) _addcarry_u64(0, base, addend, sum) -#define add64carry_next(carry, base, addend, sum) \ - _addcarry_u64(carry, base, addend, sum) -#define add64carry_last(carry, base, addend, sum) \ - (void)_addcarry_u64(carry, base, addend, sum) -#endif - -#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) -#pragma intrinsic(__umulh) -#define mul_64x64_high(a, b) __umulh(a, b) -#endif - -#if defined(_M_IX86) -#pragma intrinsic(__emulu) -#define mul_32x32_64(a, b) __emulu(a, b) + #if _MSC_FULL_VER < 190024234 && defined(_M_IX86) + #pragma message( \ + "For AES-NI at least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required.") + #endif + #if _MSC_FULL_VER < 191526730 + #pragma message( \ + "It is recommended to use \"Microsoft C/C++ Compiler\" version 19.15.26730 (Visual Studio 2017 15.8) or newer.") + #endif + #if _MSC_FULL_VER < 180040629 + #error At least "Microsoft C/C++ Compiler" version 18.00.40629 (Visual Studio 2013 Update 5) is required. + #endif + + #pragma warning(push, 1) + + #include + #include + #define likely(cond) (cond) + #define unlikely(cond) (cond) + #define unreachable() __assume(0) + #define bswap64(v) _byteswap_uint64(v) + #define bswap32(v) _byteswap_ulong(v) + #define bswap16(v) _byteswap_ushort(v) + #define rot64(v, s) _rotr64(v, s) + #define rot32(v, s) _rotr(v, s) + #define __always_inline __forceinline + + #if defined(_M_X64) || defined(_M_IA64) + #pragma intrinsic(_umul128) + #define mul_64x64_128(a, b, ph) _umul128(a, b, ph) + #pragma intrinsic(_addcarry_u64) + #define add64carry_first(base, addend, sum) \ + _addcarry_u64(0, base, addend, sum) + #define add64carry_next(carry, base, addend, sum) \ + _addcarry_u64(carry, base, addend, sum) + #define add64carry_last(carry, base, addend, sum) \ + (void)_addcarry_u64(carry, base, addend, sum) + #endif + + #if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + #pragma intrinsic(__umulh) + #define mul_64x64_high(a, b) __umulh(a, b) + #endif + + #if defined(_M_IX86) + #pragma intrinsic(__emulu) + #define mul_32x32_64(a, b) __emulu(a, b) + + #if _MSC_VER >= 1915 /* LY: workaround for SSA-optimizer bug */ + #pragma intrinsic(_addcarry_u32) + #define add32carry_first(base, addend, sum) \ + _addcarry_u32(0, base, addend, sum) + #define add32carry_next(carry, base, addend, sum) \ + _addcarry_u32(carry, base, addend, sum) + #define add32carry_last(carry, base, addend, sum) \ + (void)_addcarry_u32(carry, base, addend, sum) + +static __forceinline char msvc32_add64carry_first(uint64_t base, + uint64_t addend, + uint64_t *sum) { -#if _MSC_VER >= 1915 /* LY: workaround for SSA-optimizer bug */ -#pragma intrinsic(_addcarry_u32) -#define add32carry_first(base, addend, sum) _addcarry_u32(0, base, addend, sum) -#define add32carry_next(carry, base, addend, sum) \ - _addcarry_u32(carry, base, addend, sum) -#define add32carry_last(carry, base, addend, sum) \ - (void)_addcarry_u32(carry, base, addend, sum) - -static __forceinline char -msvc32_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) { uint32_t *const sum32 = (uint32_t *)sum; - const uint32_t base_32l = (uint32_t)base; - const uint32_t base_32h = (uint32_t)(base >> 32); - const uint32_t addend_32l = (uint32_t)addend; - const uint32_t addend_32h = (uint32_t)(addend >> 32); + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); return add32carry_next(add32carry_first(base_32l, addend_32l, sum32), base_32h, addend_32h, sum32 + 1); + } -#define add64carry_first(base, addend, sum) \ - msvc32_add64carry_first(base, addend, sum) +\ + #define add64carry_first(base, addend, sum) \ + msvc32_add64carry_first(base, addend, sum) static __forceinline char msvc32_add64carry_next(char carry, uint64_t base, - uint64_t addend, + uint64_t addend, uint64_t *sum) { + uint32_t *const sum32 = (uint32_t *)sum; - const uint32_t base_32l = (uint32_t)base; - const uint32_t base_32h = (uint32_t)(base >> 32); - const uint32_t addend_32l = (uint32_t)addend; - const uint32_t addend_32h = (uint32_t)(addend >> 32); + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); return add32carry_next(add32carry_next(carry, base_32l, addend_32l, sum32), base_32h, addend_32h, sum32 + 1); + } -#define add64carry_next(carry, base, addend, sum) \ - msvc32_add64carry_next(carry, base, addend, sum) +\ + #define add64carry_next(carry, base, addend, sum) \ + msvc32_add64carry_next(carry, base, addend, sum) static __forceinline void msvc32_add64carry_last(char carry, uint64_t base, - uint64_t addend, + uint64_t addend, uint64_t *sum) { + uint32_t *const sum32 = (uint32_t *)sum; - const uint32_t base_32l = (uint32_t)base; - const uint32_t base_32h = (uint32_t)(base >> 32); - const uint32_t addend_32l = (uint32_t)addend; - const uint32_t addend_32h = (uint32_t)(addend >> 32); + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); add32carry_last(add32carry_next(carry, base_32l, addend_32l, sum32), base_32h, addend_32h, sum32 + 1); -} -#define add64carry_last(carry, base, addend, sum) \ - msvc32_add64carry_last(carry, base, addend, sum) -#endif /* _MSC_FULL_VER >= 190024231 */ - -#elif defined(_M_ARM) -#define mul_32x32_64(a, b) _arm_umull(a, b) -#endif -#pragma warning(pop) -#pragma warning(disable : 4514) /* 'xyz': unreferenced inline function \ - has been removed */ -#pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for \ - automatic inline expansion */ -#pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4702) /* unreachable code */ -#endif /* Compiler */ +} +\ + #define add64carry_last(carry, base, addend, sum) \ + msvc32_add64carry_last(carry, base, addend, sum) + #endif /* _MSC_FULL_VER >= 190024231 */ + + #elif defined(_M_ARM) + #define mul_32x32_64(a, b) _arm_umull(a, b) + #endif + + #pragma warning(pop) + #pragma warning(disable : 4514) /* 'xyz': unreferenced inline function \ + has been removed */ + #pragma warning(disable : 4710) /* 'xyz': function not inlined */ + #pragma warning(disable : 4711) /* function 'xyz' selected for \ + automatic inline expansion */ + #pragma warning(disable : 4127) /* conditional expression is constant */ + #pragma warning(disable : 4702) /* unreachable code */ +#endif /* Compiler */ #ifndef likely -#define likely(cond) (cond) + #define likely(cond) (cond) #endif #ifndef unlikely -#define unlikely(cond) (cond) + #define unlikely(cond) (cond) #endif #ifndef __maybe_unused -#define __maybe_unused + #define __maybe_unused #endif #ifndef __always_inline -#define __always_inline __inline + #define __always_inline __inline #endif #ifndef unreachable -#define unreachable() \ - do { \ - } while (1) + #define unreachable() \ + do { \ + \ + } while (1) #endif #ifndef bswap64 -#if defined(bswap_64) -#define bswap64 bswap_64 -#elif defined(__bswap_64) -#define bswap64 __bswap_64 -#else + #if defined(bswap_64) + #define bswap64 bswap_64 + #elif defined(__bswap_64) + #define bswap64 __bswap_64 + #else static __always_inline uint64_t bswap64(uint64_t v) { + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); + } -#endif -#endif /* bswap64 */ + + #endif +#endif /* bswap64 */ #ifndef bswap32 -#if defined(bswap_32) -#define bswap32 bswap_32 -#elif defined(__bswap_32) -#define bswap32 __bswap_32 -#else + #if defined(bswap_32) + #define bswap32 bswap_32 + #elif defined(__bswap_32) + #define bswap32 __bswap_32 + #else static __always_inline uint32_t bswap32(uint32_t v) { + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); + } -#endif -#endif /* bswap32 */ + + #endif +#endif /* bswap32 */ #ifndef bswap16 -#if defined(bswap_16) -#define bswap16 bswap_16 -#elif defined(__bswap_16) -#define bswap16 __bswap_16 -#else -static __always_inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; } -#endif -#endif /* bswap16 */ + #if defined(bswap_16) + #define bswap16 bswap_16 + #elif defined(__bswap_16) + #define bswap16 __bswap_16 + #else +static __always_inline uint16_t bswap16(uint16_t v) { + + return v << 8 | v >> 8; + +} + + #endif +#endif /* bswap16 */ -#if defined(__ia32__) || \ +#if defined(__ia32__) || \ T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT -/* The __builtin_assume_aligned() leads gcc/clang to load values into the - * registers, even when it is possible to directly use an operand from memory. - * This can lead to a shortage of registers and a significant slowdown. - * Therefore avoid unnecessary use of __builtin_assume_aligned() for x86. */ -#define read_unaligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) -#define read_aligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) -#endif /* __ia32__ */ + /* The __builtin_assume_aligned() leads gcc/clang to load values into the + * registers, even when it is possible to directly use an operand from memory. + * This can lead to a shortage of registers and a significant slowdown. + * Therefore avoid unnecessary use of __builtin_assume_aligned() for x86. */ + #define read_unaligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) + #define read_aligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) +#endif /* __ia32__ */ #ifndef read_unaligned -#if defined(__GNUC__) || __has_attribute(__packed__) + #if defined(__GNUC__) || __has_attribute(__packed__) typedef struct { - uint8_t unaligned_8; + + uint8_t unaligned_8; uint16_t unaligned_16; uint32_t unaligned_32; uint64_t unaligned_64; + } __attribute__((__packed__)) t1ha_unaligned_proxy; -#define read_unaligned(ptr, bits) \ - (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ - t1ha_unaligned_proxy, unaligned_##bits))) \ - ->unaligned_##bits) -#elif defined(_MSC_VER) -#pragma warning( \ - disable : 4235) /* nonstandard extension used: '__unaligned' \ - * keyword not supported on this architecture */ -#define read_unaligned(ptr, bits) (*(const __unaligned uint##bits##_t *)(ptr)) -#else -#pragma pack(push, 1) +\ + #define read_unaligned(ptr, bits) \ + (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ + t1ha_unaligned_proxy, unaligned_##bits))) \ + ->unaligned_##bits) + #elif defined(_MSC_VER) + #pragma warning( \ + disable : 4235) /* nonstandard extension used: '__unaligned' \ + * keyword not supported on this architecture */ + #define read_unaligned(ptr, bits) \ + (*(const __unaligned uint##bits##_t *)(ptr)) + #else + #pragma pack(push, 1) typedef struct { - uint8_t unaligned_8; + + uint8_t unaligned_8; uint16_t unaligned_16; uint32_t unaligned_32; uint64_t unaligned_64; + } t1ha_unaligned_proxy; -#pragma pack(pop) -#define read_unaligned(ptr, bits) \ - (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ - t1ha_unaligned_proxy, unaligned_##bits))) \ - ->unaligned_##bits) -#endif -#endif /* read_unaligned */ + + #pragma pack(pop) + #define read_unaligned(ptr, bits) \ + (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ + t1ha_unaligned_proxy, unaligned_##bits))) \ + ->unaligned_##bits) + #endif +#endif /* read_unaligned */ #ifndef read_aligned -#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned) -#define read_aligned(ptr, bits) \ - (*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits)) -#elif (__GNUC_PREREQ(3, 3) || __has_attribute(__aligned__)) && \ - !defined(__clang__) -#define read_aligned(ptr, bits) \ - (*(const uint##bits##_t \ - __attribute__((__aligned__(ALIGNMENT_##bits))) *)(ptr)) -#elif __has_attribute(__assume_aligned__) - -static __always_inline const - uint16_t *__attribute__((__assume_aligned__(ALIGNMENT_16))) - cast_aligned_16(const void *ptr) { + #if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned) + #define read_aligned(ptr, bits) \ + (*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits)) + #elif (__GNUC_PREREQ(3, 3) || __has_attribute(__aligned__)) && \ + !defined(__clang__) + #define read_aligned(ptr, bits) \ + (*(const uint##bits##_t \ + __attribute__((__aligned__(ALIGNMENT_##bits))) *)(ptr)) + #elif __has_attribute(__assume_aligned__) + +static __always_inline const uint16_t *__attribute__(( + __assume_aligned__(ALIGNMENT_16))) cast_aligned_16(const void *ptr) { + return (const uint16_t *)ptr; + } -static __always_inline const - uint32_t *__attribute__((__assume_aligned__(ALIGNMENT_32))) - cast_aligned_32(const void *ptr) { + +static __always_inline const uint32_t *__attribute__(( + __assume_aligned__(ALIGNMENT_32))) cast_aligned_32(const void *ptr) { + return (const uint32_t *)ptr; + } -static __always_inline const - uint64_t *__attribute__((__assume_aligned__(ALIGNMENT_64))) - cast_aligned_64(const void *ptr) { + +static __always_inline const uint64_t *__attribute__(( + __assume_aligned__(ALIGNMENT_64))) cast_aligned_64(const void *ptr) { + return (const uint64_t *)ptr; + } -#define read_aligned(ptr, bits) (*cast_aligned_##bits(ptr)) + #define read_aligned(ptr, bits) (*cast_aligned_##bits(ptr)) -#elif defined(_MSC_VER) -#define read_aligned(ptr, bits) \ - (*(const __declspec(align(ALIGNMENT_##bits)) uint##bits##_t *)(ptr)) -#else -#define read_aligned(ptr, bits) (*(const uint##bits##_t *)(ptr)) -#endif -#endif /* read_aligned */ + #elif defined(_MSC_VER) + #define read_aligned(ptr, bits) \ + (*(const __declspec(align(ALIGNMENT_##bits)) uint##bits##_t *)(ptr)) + #else + #define read_aligned(ptr, bits) (*(const uint##bits##_t *)(ptr)) + #endif +#endif /* read_aligned */ #ifndef prefetch -#if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \ - !defined(__ia32__) -#define prefetch(ptr) __builtin_prefetch(ptr) -#elif defined(_M_ARM64) || defined(_M_ARM) -#define prefetch(ptr) __prefetch(ptr) -#else -#define prefetch(ptr) \ - do { \ - (void)(ptr); \ - } while (0) -#endif -#endif /* prefetch */ + #if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \ + !defined(__ia32__) + #define prefetch(ptr) __builtin_prefetch(ptr) + #elif defined(_M_ARM64) || defined(_M_ARM) + #define prefetch(ptr) __prefetch(ptr) + #else + #define prefetch(ptr) \ + do { \ + \ + (void)(ptr); \ + \ + } while (0) + #endif +#endif /* prefetch */ #if __has_warning("-Wconstant-logical-operand") -#if defined(__clang__) -#pragma clang diagnostic ignored "-Wconstant-logical-operand" -#elif defined(__GNUC__) -#pragma GCC diagnostic ignored "-Wconstant-logical-operand" -#else -#pragma warning disable "constant-logical-operand" -#endif -#endif /* -Wconstant-logical-operand */ + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wconstant-logical-operand" + #elif defined(__GNUC__) + #pragma GCC diagnostic ignored "-Wconstant-logical-operand" + #else + #pragma warning disable "constant-logical-operand" + #endif +#endif /* -Wconstant-logical-operand */ #if __has_warning("-Wtautological-pointer-compare") -#if defined(__clang__) -#pragma clang diagnostic ignored "-Wtautological-pointer-compare" -#elif defined(__GNUC__) -#pragma GCC diagnostic ignored "-Wtautological-pointer-compare" -#else -#pragma warning disable "tautological-pointer-compare" -#endif -#endif /* -Wtautological-pointer-compare */ + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wtautological-pointer-compare" + #elif defined(__GNUC__) + #pragma GCC diagnostic ignored "-Wtautological-pointer-compare" + #else + #pragma warning disable "tautological-pointer-compare" + #endif +#endif /* -Wtautological-pointer-compare */ /***************************************************************************/ #if __GNUC_PREREQ(4, 0) -#pragma GCC visibility push(hidden) -#endif /* __GNUC_PREREQ(4,0) */ + #pragma GCC visibility push(hidden) +#endif /* __GNUC_PREREQ(4,0) */ /*---------------------------------------------------------- Little Endian */ #ifndef fetch16_le_aligned static __maybe_unused __always_inline uint16_t fetch16_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_aligned(v, 16); -#else + #else return bswap16(read_aligned(v, 16)); -#endif + #endif + } -#endif /* fetch16_le_aligned */ + +#endif /* fetch16_le_aligned */ #ifndef fetch16_le_unaligned static __maybe_unused __always_inline uint16_t fetch16_le_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE const uint8_t *p = (const uint8_t *)v; return p[0] | (uint16_t)p[1] << 8; -#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_unaligned(v, 16); -#else + #else return bswap16(read_unaligned(v, 16)); -#endif + #endif + } -#endif /* fetch16_le_unaligned */ + +#endif /* fetch16_le_unaligned */ #ifndef fetch32_le_aligned static __maybe_unused __always_inline uint32_t fetch32_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_aligned(v, 32); -#else + #else return bswap32(read_aligned(v, 32)); -#endif + #endif + } -#endif /* fetch32_le_aligned */ + +#endif /* fetch32_le_aligned */ #ifndef fetch32_le_unaligned static __maybe_unused __always_inline uint32_t fetch32_le_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE return fetch16_le_unaligned(v) | (uint32_t)fetch16_le_unaligned((const uint8_t *)v + 2) << 16; -#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_unaligned(v, 32); -#else + #else return bswap32(read_unaligned(v, 32)); -#endif + #endif + } -#endif /* fetch32_le_unaligned */ + +#endif /* fetch32_le_unaligned */ #ifndef fetch64_le_aligned static __maybe_unused __always_inline uint64_t fetch64_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_aligned(v, 64); -#else + #else return bswap64(read_aligned(v, 64)); -#endif + #endif + } -#endif /* fetch64_le_aligned */ + +#endif /* fetch64_le_aligned */ #ifndef fetch64_le_unaligned static __maybe_unused __always_inline uint64_t fetch64_le_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE return fetch32_le_unaligned(v) | (uint64_t)fetch32_le_unaligned((const uint8_t *)v + 4) << 32; -#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_unaligned(v, 64); -#else + #else return bswap64(read_unaligned(v, 64)); -#endif + #endif + } -#endif /* fetch64_le_unaligned */ + +#endif /* fetch64_le_unaligned */ static __maybe_unused __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) { + const uint8_t *const p = (const uint8_t *)v; #if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__) /* We can perform a 'oneshot' read, which is little bit faster. */ @@ -611,79 +677,84 @@ static __maybe_unused __always_inline uint64_t tail64_le_aligned(const void *v, #else uint64_t r = 0; switch (tail & 7) { - default: - unreachable(); -/* fall through */ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - /* For most CPUs this code is better when not needed byte reordering. */ - case 0: - return fetch64_le_aligned(p); - case 7: - r = (uint64_t)p[6] << 8; - /* fall through */ - case 6: - r += p[5]; - r <<= 8; - /* fall through */ - case 5: - r += p[4]; - r <<= 32; - /* fall through */ - case 4: - return r + fetch32_le_aligned(p); - case 3: - r = (uint64_t)p[2] << 16; - /* fall through */ - case 2: - return r + fetch16_le_aligned(p); - case 1: - return p[0]; -#else - case 0: - r = p[7] << 8; - /* fall through */ - case 7: - r += p[6]; - r <<= 8; - /* fall through */ - case 6: - r += p[5]; - r <<= 8; - /* fall through */ - case 5: - r += p[4]; - r <<= 8; - /* fall through */ - case 4: - r += p[3]; - r <<= 8; - /* fall through */ - case 3: - r += p[2]; - r <<= 8; - /* fall through */ - case 2: - r += p[1]; - r <<= 8; + + default: + unreachable(); /* fall through */ - case 1: - return r + p[0]; -#endif + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* For most CPUs this code is better when not needed byte reordering. */ + case 0: + return fetch64_le_aligned(p); + case 7: + r = (uint64_t)p[6] << 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 32; + /* fall through */ + case 4: + return r + fetch32_le_aligned(p); + case 3: + r = (uint64_t)p[2] << 16; + /* fall through */ + case 2: + return r + fetch16_le_aligned(p); + case 1: + return p[0]; + #else + case 0: + r = p[7] << 8; + /* fall through */ + case 7: + r += p[6]; + r <<= 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 8; + /* fall through */ + case 4: + r += p[3]; + r <<= 8; + /* fall through */ + case 3: + r += p[2]; + r <<= 8; + /* fall through */ + case 2: + r += p[1]; + r <<= 8; + /* fall through */ + case 1: + return r + p[0]; + #endif + } -#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + } -#if T1HA_USE_FAST_ONESHOT_READ && \ - T1HA_SYS_UNALIGNED_ACCESS != T1HA_UNALIGNED_ACCESS__UNABLE && \ +#if T1HA_USE_FAST_ONESHOT_READ && \ + T1HA_SYS_UNALIGNED_ACCESS != T1HA_UNALIGNED_ACCESS__UNABLE && \ defined(PAGESIZE) && PAGESIZE > 42 && !defined(__SANITIZE_ADDRESS__) -#define can_read_underside(ptr, size) \ - (((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0) -#endif /* T1HA_USE_FAST_ONESHOT_READ */ + #define can_read_underside(ptr, size) \ + (((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0) +#endif /* T1HA_USE_FAST_ONESHOT_READ */ static __maybe_unused __always_inline uint64_t tail64_le_unaligned(const void *v, size_t tail) { + const uint8_t *p = (const uint8_t *)v; -#if defined(can_read_underside) && \ +#if defined(can_read_underside) && \ (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) /* On some systems (e.g. x86_64) we can perform a 'oneshot' read, which * is little bit faster. Thanks Marcin Żukowski @@ -691,77 +762,84 @@ tail64_le_unaligned(const void *v, size_t tail) { const unsigned offset = (8 - tail) & 7; const unsigned shift = offset << 3; if (likely(can_read_underside(p, 8))) { + p -= offset; return fetch64_le_unaligned(p) >> shift; + } + return fetch64_le_unaligned(p) & ((~UINT64_C(0)) >> shift); #else uint64_t r = 0; switch (tail & 7) { - default: - unreachable(); -/* fall through */ -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ - __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - /* For most CPUs this code is better when not needed - * copying for alignment or byte reordering. */ - case 0: - return fetch64_le_unaligned(p); - case 7: - r = (uint64_t)p[6] << 8; - /* fall through */ - case 6: - r += p[5]; - r <<= 8; - /* fall through */ - case 5: - r += p[4]; - r <<= 32; - /* fall through */ - case 4: - return r + fetch32_le_unaligned(p); - case 3: - r = (uint64_t)p[2] << 16; - /* fall through */ - case 2: - return r + fetch16_le_unaligned(p); - case 1: - return p[0]; -#else - /* For most CPUs this code is better than a - * copying for alignment and/or byte reordering. */ - case 0: - r = p[7] << 8; - /* fall through */ - case 7: - r += p[6]; - r <<= 8; - /* fall through */ - case 6: - r += p[5]; - r <<= 8; - /* fall through */ - case 5: - r += p[4]; - r <<= 8; - /* fall through */ - case 4: - r += p[3]; - r <<= 8; - /* fall through */ - case 3: - r += p[2]; - r <<= 8; - /* fall through */ - case 2: - r += p[1]; - r <<= 8; + + default: + unreachable(); /* fall through */ - case 1: - return r + p[0]; -#endif + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* For most CPUs this code is better when not needed + * copying for alignment or byte reordering. */ + case 0: + return fetch64_le_unaligned(p); + case 7: + r = (uint64_t)p[6] << 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 32; + /* fall through */ + case 4: + return r + fetch32_le_unaligned(p); + case 3: + r = (uint64_t)p[2] << 16; + /* fall through */ + case 2: + return r + fetch16_le_unaligned(p); + case 1: + return p[0]; + #else + /* For most CPUs this code is better than a + * copying for alignment and/or byte reordering. */ + case 0: + r = p[7] << 8; + /* fall through */ + case 7: + r += p[6]; + r <<= 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 8; + /* fall through */ + case 4: + r += p[3]; + r <<= 8; + /* fall through */ + case 3: + r += p[2]; + r <<= 8; + /* fall through */ + case 2: + r += p[1]; + r <<= 8; + /* fall through */ + case 1: + return r + p[0]; + #endif + } -#endif /* can_read_underside */ + +#endif /* can_read_underside */ + } /*------------------------------------------------------------- Big Endian */ @@ -769,83 +847,102 @@ tail64_le_unaligned(const void *v, size_t tail) { #ifndef fetch16_be_aligned static __maybe_unused __always_inline uint16_t fetch16_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_aligned(v, 16); -#else + #else return bswap16(read_aligned(v, 16)); -#endif + #endif + } -#endif /* fetch16_be_aligned */ + +#endif /* fetch16_be_aligned */ #ifndef fetch16_be_unaligned static __maybe_unused __always_inline uint16_t fetch16_be_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE const uint8_t *p = (const uint8_t *)v; return (uint16_t)p[0] << 8 | p[1]; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_unaligned(v, 16); -#else + #else return bswap16(read_unaligned(v, 16)); -#endif + #endif + } -#endif /* fetch16_be_unaligned */ + +#endif /* fetch16_be_unaligned */ #ifndef fetch32_be_aligned static __maybe_unused __always_inline uint32_t fetch32_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_aligned(v, 32); -#else + #else return bswap32(read_aligned(v, 32)); -#endif + #endif + } -#endif /* fetch32_be_aligned */ + +#endif /* fetch32_be_aligned */ #ifndef fetch32_be_unaligned static __maybe_unused __always_inline uint32_t fetch32_be_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE return (uint32_t)fetch16_be_unaligned(v) << 16 | fetch16_be_unaligned((const uint8_t *)v + 2); -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_unaligned(v, 32); -#else + #else return bswap32(read_unaligned(v, 32)); -#endif + #endif + } -#endif /* fetch32_be_unaligned */ + +#endif /* fetch32_be_unaligned */ #ifndef fetch64_be_aligned static __maybe_unused __always_inline uint64_t fetch64_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_aligned(v, 64); -#else + #else return bswap64(read_aligned(v, 64)); -#endif + #endif + } -#endif /* fetch64_be_aligned */ + +#endif /* fetch64_be_aligned */ #ifndef fetch64_be_unaligned static __maybe_unused __always_inline uint64_t fetch64_be_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE return (uint64_t)fetch32_be_unaligned(v) << 32 | fetch32_be_unaligned((const uint8_t *)v + 4); -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_unaligned(v, 64); -#else + #else return bswap64(read_unaligned(v, 64)); -#endif + #endif + } -#endif /* fetch64_be_unaligned */ + +#endif /* fetch64_be_unaligned */ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, size_t tail) { + const uint8_t *const p = (const uint8_t *)v; #if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__) /* We can perform a 'oneshot' read, which is little bit faster. */ @@ -853,61 +950,66 @@ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, return fetch64_be_aligned(p) >> shift; #else switch (tail & 7) { - default: - unreachable(); -/* fall through */ -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - /* For most CPUs this code is better when not byte reordering. */ - case 1: - return p[0]; - case 2: - return fetch16_be_aligned(p); - case 3: - return (uint32_t)fetch16_be_aligned(p) << 8 | p[2]; - case 4: - return fetch32_be_aligned(p); - case 5: - return (uint64_t)fetch32_be_aligned(p) << 8 | p[4]; - case 6: - return (uint64_t)fetch32_be_aligned(p) << 16 | fetch16_be_aligned(p + 4); - case 7: - return (uint64_t)fetch32_be_aligned(p) << 24 | - (uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6]; - case 0: - return fetch64_be_aligned(p); -#else - case 1: - return p[0]; - case 2: - return p[1] | (uint32_t)p[0] << 8; - case 3: - return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; - case 4: - return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | - (uint32_t)p[0] << 24; - case 5: - return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | - (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; - case 6: - return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | - (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; - case 7: - return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | - (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 | - (uint64_t)p[0] << 48; - case 0: - return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | - (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 | - (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; -#endif + + default: + unreachable(); + /* fall through */ + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + /* For most CPUs this code is better when not byte reordering. */ + case 1: + return p[0]; + case 2: + return fetch16_be_aligned(p); + case 3: + return (uint32_t)fetch16_be_aligned(p) << 8 | p[2]; + case 4: + return fetch32_be_aligned(p); + case 5: + return (uint64_t)fetch32_be_aligned(p) << 8 | p[4]; + case 6: + return (uint64_t)fetch32_be_aligned(p) << 16 | fetch16_be_aligned(p + 4); + case 7: + return (uint64_t)fetch32_be_aligned(p) << 24 | + (uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6]; + case 0: + return fetch64_be_aligned(p); + #else + case 1: + return p[0]; + case 2: + return p[1] | (uint32_t)p[0] << 8; + case 3: + return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; + case 4: + return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | + (uint32_t)p[0] << 24; + case 5: + return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | + (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; + case 6: + return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | + (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; + case 7: + return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | + (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | + (uint64_t)p[1] << 40 | (uint64_t)p[0] << 48; + case 0: + return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | + (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | + (uint64_t)p[2] << 40 | (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; + #endif + } -#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + } static __maybe_unused __always_inline uint64_t tail64_be_unaligned(const void *v, size_t tail) { + const uint8_t *p = (const uint8_t *)v; -#if defined(can_read_underside) && \ +#if defined(can_read_underside) && \ (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) /* On some systems (e.g. x86_64) we can perform a 'oneshot' read, which * is little bit faster. Thanks Marcin Żukowski @@ -915,139 +1017,167 @@ tail64_be_unaligned(const void *v, size_t tail) { const unsigned offset = (8 - tail) & 7; const unsigned shift = offset << 3; if (likely(can_read_underside(p, 8))) { + p -= offset; return fetch64_be_unaligned(p) & ((~UINT64_C(0)) >> shift); + } + return fetch64_be_unaligned(p) >> shift; #else switch (tail & 7) { - default: - unreachable(); -/* fall through */ -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ - __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - /* For most CPUs this code is better when not needed - * copying for alignment or byte reordering. */ - case 1: - return p[0]; - case 2: - return fetch16_be_unaligned(p); - case 3: - return (uint32_t)fetch16_be_unaligned(p) << 8 | p[2]; - case 4: - return fetch32_be(p); - case 5: - return (uint64_t)fetch32_be_unaligned(p) << 8 | p[4]; - case 6: - return (uint64_t)fetch32_be_unaligned(p) << 16 | - fetch16_be_unaligned(p + 4); - case 7: - return (uint64_t)fetch32_be_unaligned(p) << 24 | - (uint32_t)fetch16_be_unaligned(p + 4) << 8 | p[6]; - case 0: - return fetch64_be_unaligned(p); -#else - /* For most CPUs this code is better than a - * copying for alignment and/or byte reordering. */ - case 1: - return p[0]; - case 2: - return p[1] | (uint32_t)p[0] << 8; - case 3: - return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; - case 4: - return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | - (uint32_t)p[0] << 24; - case 5: - return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | - (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; - case 6: - return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | - (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; - case 7: - return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | - (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 | - (uint64_t)p[0] << 48; - case 0: - return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | - (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 | - (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; -#endif + + default: + unreachable(); + /* fall through */ + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + /* For most CPUs this code is better when not needed + * copying for alignment or byte reordering. */ + case 1: + return p[0]; + case 2: + return fetch16_be_unaligned(p); + case 3: + return (uint32_t)fetch16_be_unaligned(p) << 8 | p[2]; + case 4: + return fetch32_be(p); + case 5: + return (uint64_t)fetch32_be_unaligned(p) << 8 | p[4]; + case 6: + return (uint64_t)fetch32_be_unaligned(p) << 16 | + fetch16_be_unaligned(p + 4); + case 7: + return (uint64_t)fetch32_be_unaligned(p) << 24 | + (uint32_t)fetch16_be_unaligned(p + 4) << 8 | p[6]; + case 0: + return fetch64_be_unaligned(p); + #else + /* For most CPUs this code is better than a + * copying for alignment and/or byte reordering. */ + case 1: + return p[0]; + case 2: + return p[1] | (uint32_t)p[0] << 8; + case 3: + return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; + case 4: + return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | + (uint32_t)p[0] << 24; + case 5: + return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | + (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; + case 6: + return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | + (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; + case 7: + return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | + (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | + (uint64_t)p[1] << 40 | (uint64_t)p[0] << 48; + case 0: + return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | + (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | + (uint64_t)p[2] << 40 | (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; + #endif + } -#endif /* can_read_underside */ + +#endif /* can_read_underside */ + } /***************************************************************************/ #ifndef rot64 static __maybe_unused __always_inline uint64_t rot64(uint64_t v, unsigned s) { + return (v >> s) | (v << (64 - s)); + } -#endif /* rot64 */ + +#endif /* rot64 */ #ifndef mul_32x32_64 static __maybe_unused __always_inline uint64_t mul_32x32_64(uint32_t a, uint32_t b) { + return a * (uint64_t)b; + } -#endif /* mul_32x32_64 */ + +#endif /* mul_32x32_64 */ #ifndef add64carry_first -static __maybe_unused __always_inline unsigned -add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) { -#if __has_builtin(__builtin_addcll) +static __maybe_unused __always_inline unsigned add64carry_first(uint64_t base, + uint64_t addend, + uint64_t *sum) { + + #if __has_builtin(__builtin_addcll) unsigned long long carryout; *sum = __builtin_addcll(base, addend, 0, &carryout); return (unsigned)carryout; -#else + #else *sum = base + addend; return *sum < addend; -#endif /* __has_builtin(__builtin_addcll) */ + #endif /* __has_builtin(__builtin_addcll) */ + } -#endif /* add64carry_fist */ + +#endif /* add64carry_fist */ #ifndef add64carry_next -static __maybe_unused __always_inline unsigned -add64carry_next(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) { -#if __has_builtin(__builtin_addcll) +static __maybe_unused __always_inline unsigned add64carry_next(unsigned carry, + uint64_t base, + uint64_t addend, + uint64_t *sum) { + + #if __has_builtin(__builtin_addcll) unsigned long long carryout; *sum = __builtin_addcll(base, addend, carry, &carryout); return (unsigned)carryout; -#else + #else *sum = base + addend + carry; return *sum < addend || (carry && *sum == addend); -#endif /* __has_builtin(__builtin_addcll) */ + #endif /* __has_builtin(__builtin_addcll) */ + } -#endif /* add64carry_next */ + +#endif /* add64carry_next */ #ifndef add64carry_last -static __maybe_unused __always_inline void -add64carry_last(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) { -#if __has_builtin(__builtin_addcll) +static __maybe_unused __always_inline void add64carry_last(unsigned carry, + uint64_t base, + uint64_t addend, + uint64_t *sum) { + + #if __has_builtin(__builtin_addcll) unsigned long long carryout; *sum = __builtin_addcll(base, addend, carry, &carryout); (void)carryout; -#else + #else *sum = base + addend + carry; -#endif /* __has_builtin(__builtin_addcll) */ + #endif /* __has_builtin(__builtin_addcll) */ + } -#endif /* add64carry_last */ + +#endif /* add64carry_last */ #ifndef mul_64x64_128 -static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a, - uint64_t b, +static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a, + uint64_t b, uint64_t *h) { -#if (defined(__SIZEOF_INT128__) || \ - (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) && \ - (!defined(__LCC__) || __LCC__ != 124) + + #if (defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) && \ + (!defined(__LCC__) || __LCC__ != 124) __uint128_t r = (__uint128_t)a * (__uint128_t)b; /* modern GCC could nicely optimize this */ *h = (uint64_t)(r >> 64); return (uint64_t)r; -#elif defined(mul_64x64_high) + #elif defined(mul_64x64_high) *h = mul_64x64_high(a, b); return a * b; -#else + #else /* performs 64x64 to 128 bit multiplication */ const uint64_t ll = mul_32x32_64((uint32_t)a, (uint32_t)b); const uint64_t lh = mul_32x32_64(a >> 32, (uint32_t)b); @@ -1062,18 +1192,23 @@ static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a, add64carry_last(add64carry_first(ll, lh << 32, &l), hh, lh >> 32, h); add64carry_last(add64carry_first(l, hl << 32, &l), *h, hl >> 32, h); return l; -#endif + #endif + } -#endif /* mul_64x64_128() */ + +#endif /* mul_64x64_128() */ #ifndef mul_64x64_high static __maybe_unused __always_inline uint64_t mul_64x64_high(uint64_t a, uint64_t b) { + uint64_t h; mul_64x64_128(a, b, &h); return h; + } -#endif /* mul_64x64_high */ + +#endif /* mul_64x64_high */ /***************************************************************************/ @@ -1089,45 +1224,56 @@ static const uint64_t prime_6 = UINT64_C(0xCB5AF53AE3AAAC31); /* xor high and low parts of full 128-bit product */ static __maybe_unused __always_inline uint64_t mux64(uint64_t v, uint64_t prime) { + uint64_t l, h; l = mul_64x64_128(v, prime, &h); return l ^ h; + } static __maybe_unused __always_inline uint64_t final64(uint64_t a, uint64_t b) { + uint64_t x = (a + rot64(b, 41)) * prime_0; uint64_t y = (rot64(a, 23) + b) * prime_6; return mux64(x ^ y, prime_5); + } static __maybe_unused __always_inline void mixup64(uint64_t *__restrict a, uint64_t *__restrict b, uint64_t v, uint64_t prime) { + uint64_t h; *a ^= mul_64x64_128(*b + v, prime, &h); *b += h; + } /***************************************************************************/ typedef union t1ha_uint128 { -#if defined(__SIZEOF_INT128__) || \ + +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) __uint128_t v; #endif struct { + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ uint64_t l, h; #else uint64_t h, l; #endif + }; + } t1ha_uint128_t; static __maybe_unused __always_inline t1ha_uint128_t not128(const t1ha_uint128_t v) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = ~v.v; #else @@ -1135,13 +1281,15 @@ not128(const t1ha_uint128_t v) { r.h = ~v.h; #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t left128(const t1ha_uint128_t v, unsigned s) { + t1ha_uint128_t r; assert(s < 128); -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = v.v << s; #else @@ -1149,13 +1297,15 @@ left128(const t1ha_uint128_t v, unsigned s) { r.h = (s < 64) ? (v.h << s) | (s ? v.l >> (64 - s) : 0) : v.l << (s - 64); #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t right128(const t1ha_uint128_t v, unsigned s) { + t1ha_uint128_t r; assert(s < 128); -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = v.v >> s; #else @@ -1163,12 +1313,14 @@ right128(const t1ha_uint128_t v, unsigned s) { r.h = (s < 64) ? v.h >> s : 0; #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t or128(t1ha_uint128_t x, t1ha_uint128_t y) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = x.v | y.v; #else @@ -1176,12 +1328,14 @@ static __maybe_unused __always_inline t1ha_uint128_t or128(t1ha_uint128_t x, r.h = x.h | y.h; #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t xor128(t1ha_uint128_t x, t1ha_uint128_t y) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = x.v ^ y.v; #else @@ -1189,36 +1343,42 @@ static __maybe_unused __always_inline t1ha_uint128_t xor128(t1ha_uint128_t x, r.h = x.h ^ y.h; #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t rot128(t1ha_uint128_t v, - unsigned s) { + unsigned s) { + s &= 127; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) v.v = (v.v << (128 - s)) | (v.v >> s); return v; #else return s ? or128(left128(v, 128 - s), right128(v, s)) : v; #endif + } static __maybe_unused __always_inline t1ha_uint128_t add128(t1ha_uint128_t x, t1ha_uint128_t y) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = x.v + y.v; #else add64carry_last(add64carry_first(x.l, y.l, &r.l), x.h, y.h, &r.h); #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x, t1ha_uint128_t y) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = x.v * y.v; #else @@ -1226,6 +1386,7 @@ static __maybe_unused __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x, r.h += x.l * y.h + y.l * x.h; #endif return r; + } /***************************************************************************/ @@ -1233,22 +1394,29 @@ static __maybe_unused __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x, #if T1HA0_AESNI_AVAILABLE && defined(__ia32__) uint64_t t1ha_ia32cpu_features(void); -static __maybe_unused __always_inline bool -t1ha_ia32_AESNI_avail(uint64_t ia32cpu_features) { +static __maybe_unused __always_inline bool t1ha_ia32_AESNI_avail( + uint64_t ia32cpu_features) { + /* check for AES-NI */ return (ia32cpu_features & UINT32_C(0x02000000)) != 0; + } -static __maybe_unused __always_inline bool -t1ha_ia32_AVX_avail(uint64_t ia32cpu_features) { +static __maybe_unused __always_inline bool t1ha_ia32_AVX_avail( + uint64_t ia32cpu_features) { + /* check for any AVX */ return (ia32cpu_features & UINT32_C(0x1A000000)) == UINT32_C(0x1A000000); + } -static __maybe_unused __always_inline bool -t1ha_ia32_AVX2_avail(uint64_t ia32cpu_features) { +static __maybe_unused __always_inline bool t1ha_ia32_AVX2_avail( + uint64_t ia32cpu_features) { + /* check for 'Advanced Vector Extensions 2' */ return ((ia32cpu_features >> 32) & 32) != 0; + } -#endif /* T1HA0_AESNI_AVAILABLE && __ia32__ */ +#endif /* T1HA0_AESNI_AVAILABLE && __ia32__ */ + diff --git a/include/t1ha_selfcheck.h b/include/t1ha_selfcheck.h index ff7c589c..65343bfe 100644 --- a/include/t1ha_selfcheck.h +++ b/include/t1ha_selfcheck.h @@ -43,8 +43,8 @@ #pragma once #if defined(_MSC_VER) && _MSC_VER > 1800 -#pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif /* MSVC */ + #pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif /* MSVC */ #include "t1ha.h" /***************************************************************************/ @@ -59,18 +59,19 @@ extern const uint64_t t1ha_refval_2atonce[81]; extern const uint64_t t1ha_refval_2atonce128[81]; extern const uint64_t t1ha_refval_2stream[81]; extern const uint64_t t1ha_refval_2stream128[81]; -#endif /* T1HA2_DISABLED */ +#endif /* T1HA2_DISABLED */ #ifndef T1HA1_DISABLED extern const uint64_t t1ha_refval_64le[81]; extern const uint64_t t1ha_refval_64be[81]; -#endif /* T1HA1_DISABLED */ +#endif /* T1HA1_DISABLED */ #ifndef T1HA0_DISABLED extern const uint64_t t1ha_refval_32le[81]; extern const uint64_t t1ha_refval_32be[81]; -#if T1HA0_AESNI_AVAILABLE + #if T1HA0_AESNI_AVAILABLE extern const uint64_t t1ha_refval_ia32aes_a[81]; extern const uint64_t t1ha_refval_ia32aes_b[81]; -#endif /* T1HA0_AESNI_AVAILABLE */ -#endif /* T1HA0_DISABLED */ + #endif /* T1HA0_AESNI_AVAILABLE */ +#endif /* T1HA0_DISABLED */ + diff --git a/include/xxhash.h b/include/xxhash.h index d11f0f63..7697d0f2 100644 --- a/include/xxhash.h +++ b/include/xxhash.h @@ -36,8 +36,8 @@ /*! * @mainpage xxHash * - * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed - * limits. + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM + * speed limits. * * It is proposed in four flavors, in three families: * 1. @ref XXH32_family @@ -54,44 +54,46 @@ * Benchmarks * --- * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. - * The open source benchmark program is compiled with clang v10.0 using -O3 flag. - * - * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | - * | -------------------- | ------- | ----: | ---------------: | ------------------: | - * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | - * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | - * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | - * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | - * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | - * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | - * | RAM sequential read | | N/A | 28.0 GB/s | N/A | - * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | - * | City64 | | 64 | 22.0 GB/s | 76.6 | - * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | - * | City128 | | 128 | 21.7 GB/s | 57.7 | - * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | - * | XXH64() | | 64 | 19.4 GB/s | 71.0 | - * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | - * | Mum | | 64 | 18.0 GB/s | 67.0 | - * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | - * | XXH32() | | 32 | 9.7 GB/s | 71.9 | - * | City32 | | 32 | 9.1 GB/s | 66.0 | - * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | - * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | - * | SipHash* | | 64 | 3.0 GB/s | 43.2 | - * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | - * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | - * | FNV64 | | 64 | 1.2 GB/s | 62.7 | - * | Blake2* | | 256 | 1.1 GB/s | 5.1 | - * | SHA1* | | 160 | 0.8 GB/s | 5.6 | - * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * The open source benchmark program is compiled with clang v10.0 using -O3 + * flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data + * Velocity | | -------------------- | ------- | ----: | ---------------: | + * ------------------: | | XXH3_64bits() | @b AVX2 | 64 | 59.4 + * GB/s | 133.1 | | MeowHash | AES-NI | 128 | 58.2 + * GB/s | 52.5 | | XXH3_128bits() | @b AVX2 | 128 | 57.9 + * GB/s | 118.1 | | CLHash | PCLMUL | 64 | 37.1 + * GB/s | 58.1 | | XXH3_64bits() | @b SSE2 | 64 | 31.5 + * GB/s | 133.1 | | XXH3_128bits() | @b SSE2 | 128 | 29.6 + * GB/s | 118.1 | | RAM sequential read | | N/A | 28.0 + * GB/s | N/A | | ahash | AES-NI | 64 | 22.5 + * GB/s | 107.2 | | City64 | | 64 | 22.0 + * GB/s | 76.6 | | T1ha2 | | 64 | 22.0 + * GB/s | 99.0 | | City128 | | 128 | 21.7 + * GB/s | 57.7 | | FarmHash | AES-NI | 64 | 21.3 + * GB/s | 71.9 | | XXH64() | | 64 | 19.4 + * GB/s | 71.0 | | SpookyHash | | 64 | 19.3 + * GB/s | 53.2 | | Mum | | 64 | 18.0 + * GB/s | 67.0 | | CRC32C | SSE4.2 | 32 | 13.0 + * GB/s | 57.9 | | XXH32() | | 32 | 9.7 + * GB/s | 71.9 | | City32 | | 32 | 9.1 + * GB/s | 66.0 | | Blake3* | @b AVX2 | 256 | 4.4 + * GB/s | 8.1 | | Murmur3 | | 32 | 3.9 + * GB/s | 56.1 | | SipHash* | | 64 | 3.0 + * GB/s | 43.2 | | Blake3* | @b SSE2 | 256 | 2.4 + * GB/s | 8.1 | | HighwayHash | | 64 | 1.4 + * GB/s | 6.0 | | FNV64 | | 64 | 1.2 + * GB/s | 62.7 | | Blake2* | | 256 | 1.1 + * GB/s | 5.1 | | SHA1* | | 160 | 0.8 + * GB/s | 5.6 | | MD5* | | 128 | 0.6 + * GB/s | 7.8 | * @note - * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, - * even though it is mandatory on x64. - * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic - * by modern standards. - * - Small data velocity is a rough average of algorithm's efficiency for small - * data. For more accurate information, see the wiki. + * - Hashes which require a specific ISA extension are noted. SSE2 is also + * noted, even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is + * non-cryptographic by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for + * small data. For more accurate information, see the wiki. * - More benchmarks and strength tests are found on the wiki: * https://github.com/Cyan4973/xxHash/wiki * @@ -106,14 +108,15 @@ * - The range from [`input`, `input + length`) is valid, readable memory. * - The only exception is if the `length` is `0`, `input` may be `NULL`. * - For C++, the objects must have the *TriviallyCopyable* property, as the - * functions access bytes directly as if it was an array of `unsigned char`. + * functions access bytes directly as if it was an array of `unsigned + * char`. * * @anchor single_shot_example * **Single Shot** * - * These functions are stateless functions which hash a contiguous block of memory, - * immediately returning the result. They are the easiest and usually the fastest - * option. + * These functions are stateless functions which hash a contiguous block of + * memory, immediately returning the result. They are the easiest and usually + * the fastest option. * * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() * @@ -121,9 +124,10 @@ * #include * #include "xxhash.h" * - * // Example for a function which hashes a null terminated string with XXH32(). - * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * // Example for a function which hashes a null terminated string with + * XXH32(). XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) * { + * // NULL pointers are only valid if the length is zero * size_t length = (string == NULL) ? 0 : strlen(string); * return XXH32(string, length, seed); @@ -143,9 +147,10 @@ * #include * #include * #include "xxhash.h" - * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). - * XXH64_hash_t hashFile(FILE* f) + * // Example for a function which hashes a FILE incrementally with + * XXH3_64bits(). XXH64_hash_t hashFile(FILE* f) * { + * // Allocate a state struct. Do not just use malloc() or new. * XXH3_state_t* state = XXH3_createState(); * assert(state != NULL && "Out of memory!"); @@ -155,6 +160,7 @@ * size_t count; * // Read the file in chunks * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data * XXH3_64bits_update(state, buffer, count); * } @@ -174,7 +180,8 @@ * * Start a new hash by initializing the state with a seed using `XXH*_reset()`. * - * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * Then, feed the hash state by calling `XXH*_update()` as many times as + * necessary. * * The function returns an error code, with 0 meaning OK, and any other value * meaning there is an error. @@ -195,11 +202,13 @@ * integers. * This the simplest and fastest format for further post-processing. * - * However, this leaves open the question of what is the order on the byte level, - * since little and big endian conventions will store the same number differently. + * However, this leaves open the question of what is the order on the byte + * level, since little and big endian conventions will store the same number + * differently. * * The canonical representation settles this issue by mandating big-endian - * convention, the same convention as human-readable numbers (large digits first). + * convention, the same convention as human-readable numbers (large digits + * first). * * When writing hash values to storage, sending them over a network, or printing * them, it's highly recommended to use the canonical representation to ensure @@ -216,13 +225,15 @@ * #include * #include "xxhash.h" * - * // Example for a function which prints XXH32_hash_t in human readable format - * void printXxh32(XXH32_hash_t hash) + * // Example for a function which prints XXH32_hash_t in human readable + * format void printXxh32(XXH32_hash_t hash) * { + * XXH32_canonical_t cano; * XXH32_canonicalFromHash(&cano, hash); * size_t i; * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); * } * printf("\n"); @@ -231,6 +242,7 @@ * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); * return hash; * } @@ -241,8 +253,9 @@ * xxHash prototypes and implementation */ -#if defined (__cplusplus) +#if defined(__cplusplus) extern "C" { + #endif /* **************************** @@ -252,304 +265,328 @@ extern "C" { * @defgroup public Public API * Contains details on the public xxHash functions. * @{ - */ -#ifdef XXH_DOXYGEN -/*! - * @brief Gives access to internal state declaration, required for static allocation. - * - * Incompatible with dynamic linking, due to risks of ABI changes. - * - * Usage: - * @code{.c} - * #define XXH_STATIC_LINKING_ONLY - * #include "xxhash.h" - * @endcode - */ -# define XXH_STATIC_LINKING_ONLY -/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ - -/*! - * @brief Gives access to internal definitions. - * - * Usage: - * @code{.c} - * #define XXH_STATIC_LINKING_ONLY - * #define XXH_IMPLEMENTATION - * #include "xxhash.h" - * @endcode - */ -# define XXH_IMPLEMENTATION -/* Do not undef XXH_IMPLEMENTATION for Doxygen */ -/*! - * @brief Exposes the implementation and marks all functions as `inline`. - * - * Use these build macros to inline xxhash into the target unit. - * Inlining improves performance on small inputs, especially when the length is - * expressed as a compile-time constant: - * - * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html - * - * It also keeps xxHash symbols private to the unit, so they are not exported. - * - * Usage: - * @code{.c} - * #define XXH_INLINE_ALL - * #include "xxhash.h" - * @endcode - * Do not compile and link xxhash.o as a separate object, as it is not useful. - */ -# define XXH_INLINE_ALL -# undef XXH_INLINE_ALL -/*! - * @brief Exposes the implementation without marking functions as inline. - */ -# define XXH_PRIVATE_API -# undef XXH_PRIVATE_API -/*! - * @brief Emulate a namespace by transparently prefixing all symbols. - * - * If you want to include _and expose_ xxHash functions from within your own - * library, but also want to avoid symbol collisions with other libraries which - * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix - * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE - * (therefore, avoid empty or numeric values). - * - * Note that no change is required within the calling program as long as it - * includes `xxhash.h`: Regular symbol names will be automatically translated - * by this header. */ -# define XXH_NAMESPACE /* YOUR NAME HERE */ -# undef XXH_NAMESPACE +#ifdef XXH_DOXYGEN + /*! + * @brief Gives access to internal state declaration, required for static + * allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ + #define XXH_STATIC_LINKING_ONLY + /* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + + /*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ + #define XXH_IMPLEMENTATION + /* Do not undef XXH_IMPLEMENTATION for Doxygen */ + + /*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length + * is expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ + #define XXH_INLINE_ALL + #undef XXH_INLINE_ALL + /*! + * @brief Exposes the implementation without marking functions as inline. + */ + #define XXH_PRIVATE_API + #undef XXH_PRIVATE_API + /*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries + * which may also include xxHash, you can use @ref XXH_NAMESPACE to + * automatically prefix any public symbol from xxhash library with the value + * of @ref XXH_NAMESPACE (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ + #define XXH_NAMESPACE /* YOUR NAME HERE */ + #undef XXH_NAMESPACE #endif -#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ - && !defined(XXH_INLINE_ALL_31684351384) - /* this section should be traversed only once */ -# define XXH_INLINE_ALL_31684351384 - /* give access to the advanced API, required to compile implementations */ -# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ -# define XXH_STATIC_LINKING_ONLY - /* make all functions private */ -# undef XXH_PUBLIC_API -# if defined(__GNUC__) -# define XXH_PUBLIC_API static __inline __attribute__((unused)) -# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define XXH_PUBLIC_API static inline -# elif defined(_MSC_VER) -# define XXH_PUBLIC_API static __inline -# else - /* note: this version may generate warnings for unused static functions */ -# define XXH_PUBLIC_API static -# endif - - /* - * This part deals with the special case where a unit wants to inline xxHash, - * but "xxhash.h" has previously been included without XXH_INLINE_ALL, - * such as part of some previously included *.h header file. - * Without further action, the new include would just be ignored, - * and functions would effectively _not_ be inlined (silent failure). - * The following macros solve this situation by prefixing all inlined names, - * avoiding naming collision with previous inclusions. - */ - /* Before that, we unconditionally #undef all symbols, - * in case they were already defined with XXH_NAMESPACE. - * They will then be redefined for XXH_INLINE_ALL - */ -# undef XXH_versionNumber - /* XXH32 */ -# undef XXH32 -# undef XXH32_createState -# undef XXH32_freeState -# undef XXH32_reset -# undef XXH32_update -# undef XXH32_digest -# undef XXH32_copyState -# undef XXH32_canonicalFromHash -# undef XXH32_hashFromCanonical - /* XXH64 */ -# undef XXH64 -# undef XXH64_createState -# undef XXH64_freeState -# undef XXH64_reset -# undef XXH64_update -# undef XXH64_digest -# undef XXH64_copyState -# undef XXH64_canonicalFromHash -# undef XXH64_hashFromCanonical - /* XXH3_64bits */ -# undef XXH3_64bits -# undef XXH3_64bits_withSecret -# undef XXH3_64bits_withSeed -# undef XXH3_64bits_withSecretandSeed -# undef XXH3_createState -# undef XXH3_freeState -# undef XXH3_copyState -# undef XXH3_64bits_reset -# undef XXH3_64bits_reset_withSeed -# undef XXH3_64bits_reset_withSecret -# undef XXH3_64bits_update -# undef XXH3_64bits_digest -# undef XXH3_generateSecret - /* XXH3_128bits */ -# undef XXH128 -# undef XXH3_128bits -# undef XXH3_128bits_withSeed -# undef XXH3_128bits_withSecret -# undef XXH3_128bits_reset -# undef XXH3_128bits_reset_withSeed -# undef XXH3_128bits_reset_withSecret -# undef XXH3_128bits_reset_withSecretandSeed -# undef XXH3_128bits_update -# undef XXH3_128bits_digest -# undef XXH128_isEqual -# undef XXH128_cmp -# undef XXH128_canonicalFromHash -# undef XXH128_hashFromCanonical - /* Finally, free the namespace itself */ -# undef XXH_NAMESPACE - - /* employ the namespace for XXH_INLINE_ALL */ -# define XXH_NAMESPACE XXH_INLINE_ - /* - * Some identifiers (enums, type names) are not symbols, - * but they must nonetheless be renamed to avoid redeclaration. - * Alternative solution: do not redeclare them. - * However, this requires some #ifdefs, and has a more dispersed impact. - * Meanwhile, renaming can be achieved in a single place. - */ -# define XXH_IPREF(Id) XXH_NAMESPACE ## Id -# define XXH_OK XXH_IPREF(XXH_OK) -# define XXH_ERROR XXH_IPREF(XXH_ERROR) -# define XXH_errorcode XXH_IPREF(XXH_errorcode) -# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) -# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) -# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) -# define XXH32_state_s XXH_IPREF(XXH32_state_s) -# define XXH32_state_t XXH_IPREF(XXH32_state_t) -# define XXH64_state_s XXH_IPREF(XXH64_state_s) -# define XXH64_state_t XXH_IPREF(XXH64_state_t) -# define XXH3_state_s XXH_IPREF(XXH3_state_s) -# define XXH3_state_t XXH_IPREF(XXH3_state_t) -# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) - /* Ensure the header is parsed again, even if it was previously included */ -# undef XXHASH_H_5627135585666179 -# undef XXHASH_H_STATIC_13879238742 -#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ - -/* **************************************************************** - * Stable API - *****************************************************************/ -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - -/*! @brief Marks a global symbol. */ -#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) -# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) -# ifdef XXH_EXPORT -# define XXH_PUBLIC_API __declspec(dllexport) -# elif XXH_IMPORT -# define XXH_PUBLIC_API __declspec(dllimport) -# endif -# else -# define XXH_PUBLIC_API /* do nothing */ -# endif -#endif +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) && \ + !defined(XXH_INLINE_ALL_31684351384) +/* this section should be traversed only once */ + #define XXH_INLINE_ALL_31684351384 +/* give access to the advanced API, required to compile implementations */ + #undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ + #define XXH_STATIC_LINKING_ONLY +/* make all functions private */ + #undef XXH_PUBLIC_API + #if defined(__GNUC__) + #define XXH_PUBLIC_API static __inline __attribute__((unused)) + #elif defined(__cplusplus) || \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) + #define XXH_PUBLIC_API static inline + #elif defined(_MSC_VER) + #define XXH_PUBLIC_API static __inline + #else + /* note: this version may generate warnings for unused static functions */ + #define XXH_PUBLIC_API static + #endif -#ifdef XXH_NAMESPACE -# define XXH_CAT(A,B) A##B -# define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ +/* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ + #undef XXH_versionNumber /* XXH32 */ -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) -# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) -# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) + #undef XXH32 + #undef XXH32_createState + #undef XXH32_freeState + #undef XXH32_reset + #undef XXH32_update + #undef XXH32_digest + #undef XXH32_copyState + #undef XXH32_canonicalFromHash + #undef XXH32_hashFromCanonical /* XXH64 */ -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) -# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) -# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) -# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) + #undef XXH64 + #undef XXH64_createState + #undef XXH64_freeState + #undef XXH64_reset + #undef XXH64_update + #undef XXH64_digest + #undef XXH64_copyState + #undef XXH64_canonicalFromHash + #undef XXH64_hashFromCanonical /* XXH3_64bits */ -# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) -# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) -# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) -# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) -# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) -# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) -# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) -# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) -# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) -# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) -# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) -# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) -# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) -# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) -# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) + #undef XXH3_64bits + #undef XXH3_64bits_withSecret + #undef XXH3_64bits_withSeed + #undef XXH3_64bits_withSecretandSeed + #undef XXH3_createState + #undef XXH3_freeState + #undef XXH3_copyState + #undef XXH3_64bits_reset + #undef XXH3_64bits_reset_withSeed + #undef XXH3_64bits_reset_withSecret + #undef XXH3_64bits_update + #undef XXH3_64bits_digest + #undef XXH3_generateSecret /* XXH3_128bits */ -# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) -# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) -# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) -# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) -# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) -# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) -# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) -# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) -# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) -# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) -# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) -# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) -# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) -# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) -# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) -#endif - - -/* ************************************* -* Compiler specifics -***************************************/ - -/* specific declaration modes for Windows */ -#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) -# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) -# ifdef XXH_EXPORT -# define XXH_PUBLIC_API __declspec(dllexport) -# elif XXH_IMPORT -# define XXH_PUBLIC_API __declspec(dllimport) -# endif -# else -# define XXH_PUBLIC_API /* do nothing */ -# endif -#endif - -#if defined (__GNUC__) -# define XXH_CONSTF __attribute__((const)) -# define XXH_PUREF __attribute__((pure)) -# define XXH_MALLOCF __attribute__((malloc)) -#else -# define XXH_CONSTF /* disable */ -# define XXH_PUREF -# define XXH_MALLOCF -#endif + #undef XXH128 + #undef XXH3_128bits + #undef XXH3_128bits_withSeed + #undef XXH3_128bits_withSecret + #undef XXH3_128bits_reset + #undef XXH3_128bits_reset_withSeed + #undef XXH3_128bits_reset_withSecret + #undef XXH3_128bits_reset_withSecretandSeed + #undef XXH3_128bits_update + #undef XXH3_128bits_digest + #undef XXH128_isEqual + #undef XXH128_cmp + #undef XXH128_canonicalFromHash + #undef XXH128_hashFromCanonical +/* Finally, free the namespace itself */ + #undef XXH_NAMESPACE + +/* employ the namespace for XXH_INLINE_ALL */ + #define XXH_NAMESPACE XXH_INLINE_ +/* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ + #define XXH_IPREF(Id) XXH_NAMESPACE##Id + #define XXH_OK XXH_IPREF(XXH_OK) + #define XXH_ERROR XXH_IPREF(XXH_ERROR) + #define XXH_errorcode XXH_IPREF(XXH_errorcode) + #define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) + #define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) + #define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) + #define XXH32_state_s XXH_IPREF(XXH32_state_s) + #define XXH32_state_t XXH_IPREF(XXH32_state_t) + #define XXH64_state_s XXH_IPREF(XXH64_state_s) + #define XXH64_state_t XXH_IPREF(XXH64_state_t) + #define XXH3_state_s XXH_IPREF(XXH3_state_s) + #define XXH3_state_t XXH_IPREF(XXH3_state_t) + #define XXH128_hash_t XXH_IPREF(XXH128_hash_t) +/* Ensure the header is parsed again, even if it was previously included */ + #undef XXHASH_H_5627135585666179 + #undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ -/* ************************************* -* Version -***************************************/ -#define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 8 -#define XXH_VERSION_RELEASE 2 -/*! @brief Version number, encoded as two digits each */ -#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 + #define XXHASH_H_5627135585666179 1 + + /*! @brief Marks a global symbol. */ + #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) + #if defined(WIN32) && defined(_MSC_VER) && \ + (defined(XXH_IMPORT) || defined(XXH_EXPORT)) + #ifdef XXH_EXPORT + #define XXH_PUBLIC_API __declspec(dllexport) + #elif XXH_IMPORT + #define XXH_PUBLIC_API __declspec(dllimport) + #endif + #else + #define XXH_PUBLIC_API /* do nothing */ + #endif + #endif + + #ifdef XXH_NAMESPACE + #define XXH_CAT(A, B) A##B + #define XXH_NAME2(A, B) XXH_CAT(A, B) + #define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) + /* XXH32 */ + #define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) + #define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) + #define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) + #define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) + #define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) + #define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) + #define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) + #define XXH32_canonicalFromHash \ + XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) + #define XXH32_hashFromCanonical \ + XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) + /* XXH64 */ + #define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) + #define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) + #define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) + #define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) + #define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) + #define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) + #define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) + #define XXH64_canonicalFromHash \ + XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) + #define XXH64_hashFromCanonical \ + XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) + /* XXH3_64bits */ + #define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) + #define XXH3_64bits_withSecret \ + XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) + #define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) + #define XXH3_64bits_withSecretandSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) + #define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) + #define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) + #define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) + #define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) + #define XXH3_64bits_reset_withSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) + #define XXH3_64bits_reset_withSecret \ + XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) + #define XXH3_64bits_reset_withSecretandSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) + #define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) + #define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) + #define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) + #define XXH3_generateSecret_fromSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) + /* XXH3_128bits */ + #define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) + #define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) + #define XXH3_128bits_withSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) + #define XXH3_128bits_withSecret \ + XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) + #define XXH3_128bits_withSecretandSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) + #define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) + #define XXH3_128bits_reset_withSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) + #define XXH3_128bits_reset_withSecret \ + XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) + #define XXH3_128bits_reset_withSecretandSeed \ + XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) + #define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) + #define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) + #define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) + #define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) + #define XXH128_canonicalFromHash \ + XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) + #define XXH128_hashFromCanonical \ + XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) + #endif + + /* ************************************* + * Compiler specifics + ***************************************/ + + /* specific declaration modes for Windows */ + #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) + #if defined(WIN32) && defined(_MSC_VER) && \ + (defined(XXH_IMPORT) || defined(XXH_EXPORT)) + #ifdef XXH_EXPORT + #define XXH_PUBLIC_API __declspec(dllexport) + #elif XXH_IMPORT + #define XXH_PUBLIC_API __declspec(dllimport) + #endif + #else + #define XXH_PUBLIC_API /* do nothing */ + #endif + #endif + + #if defined(__GNUC__) + #define XXH_CONSTF __attribute__((const)) + #define XXH_PUREF __attribute__((pure)) + #define XXH_MALLOCF __attribute__((malloc)) + #else + #define XXH_CONSTF /* disable */ + #define XXH_PUREF + #define XXH_MALLOCF + #endif + + /* ************************************* + * Version + ***************************************/ + #define XXH_VERSION_MAJOR 0 + #define XXH_VERSION_MINOR 8 + #define XXH_VERSION_RELEASE 2 + /*! @brief Version number, encoded as two digits each */ + #define XXH_VERSION_NUMBER \ + (XXH_VERSION_MAJOR * 100 * 100 + XXH_VERSION_MINOR * 100 + \ + XXH_VERSION_RELEASE) /*! * @brief Obtains the xxHash version. @@ -559,26 +596,26 @@ extern "C" { * * @return @ref XXH_VERSION_NUMBER of the invoked library. */ -XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); - +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber(void); -/* **************************** -* Common basic types -******************************/ -#include /* size_t */ + /* **************************** + * Common basic types + ******************************/ + #include /* size_t */ /*! * @brief Exit code for the streaming API. */ typedef enum { - XXH_OK = 0, /*!< OK */ - XXH_ERROR /*!< Error */ -} XXH_errorcode; + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ -/*-********************************************************************** -* 32-bit hash -************************************************************************/ -#if defined(XXH_DOXYGEN) /* Don't show include */ +} XXH_errorcode; + + /*-********************************************************************** + * 32-bit hash + ************************************************************************/ + #if defined(XXH_DOXYGEN) /* Don't show include */ /*! * @brief An unsigned 32-bit integer. * @@ -586,22 +623,22 @@ typedef enum { */ typedef uint32_t XXH32_hash_t; -#elif !defined (__VMS) \ - && (defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint32_t XXH32_hash_t; - -#else -# include -# if UINT_MAX == 0xFFFFFFFFUL - typedef unsigned int XXH32_hash_t; -# elif ULONG_MAX == 0xFFFFFFFFUL - typedef unsigned long XXH32_hash_t; -# else -# error "unsupported platform: need a 32-bit type" -# endif -#endif + #elif !defined(__VMS) && \ + (defined(__cplusplus) || \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) + #include +typedef uint32_t XXH32_hash_t; + + #else + #include + #if UINT_MAX == 0xFFFFFFFFUL +typedef unsigned int XXH32_hash_t; + #elif ULONG_MAX == 0xFFFFFFFFUL +typedef unsigned long XXH32_hash_t; + #else + #error "unsupported platform: need a 32-bit type" + #endif + #endif /*! * @} @@ -618,12 +655,14 @@ typedef uint32_t XXH32_hash_t; * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families * @see @ref XXH32_impl for implementation details * @{ + */ /*! * @brief Calculates the 32-bit hash of @p input using xxHash32. * - * @param input The block of data to be hashed, at least @p length bytes in size. + * @param input The block of data to be hashed, at least @p length bytes in + * size. * @param length The length of @p input, in bytes. * @param seed The 32-bit seed to alter the hash's output predictably. * @@ -636,9 +675,10 @@ typedef uint32_t XXH32_hash_t; * * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32(const void *input, size_t length, + XXH32_hash_t seed); -#ifndef XXH_NO_STREAM + #ifndef XXH_NO_STREAM /*! * @typedef struct XXH32_state_s XXH32_state_t * @brief The opaque state struct for the XXH32 streaming API. @@ -658,11 +698,12 @@ typedef struct XXH32_state_s XXH32_state_t; * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t *XXH32_createState(void); /*! * @brief Frees an @ref XXH32_state_t. * - * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref + * XXH32_createState(). * * @return @ref XXH_OK. * @@ -671,7 +712,7 @@ XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); * @see @ref streaming_example "Streaming Example" * */ -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr); /*! * @brief Copies one @ref XXH32_state_t to another. * @@ -680,7 +721,8 @@ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); * @pre * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t *dst_state, + const XXH32_state_t *src_state); /*! * @brief Resets an @ref XXH32_state_t to begin a new hash. @@ -694,17 +736,20 @@ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_ * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * - * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + * @note This function resets and seeds a state. Call it before @ref + * XXH32_update(). * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, + XXH32_hash_t seed); /*! * @brief Consumes a block of @p input to an @ref XXH32_state_t. * * @param statePtr The state struct to update. - * @param input The block of data to be hashed, at least @p length bytes in size. + * @param input The block of data to be hashed, at least @p length bytes in + * size. * @param length The length of @p input, in bytes. * * @pre @@ -721,7 +766,8 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *statePtr, + const void *input, size_t length); /*! * @brief Returns the calculated hash value from an @ref XXH32_state_t. @@ -739,8 +785,9 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); -#endif /* !XXH_NO_STREAM */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t +XXH32_digest(const XXH32_state_t *statePtr); + #endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ @@ -748,7 +795,9 @@ XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePt * @brief Canonical (big endian) representation of @ref XXH32_hash_t. */ typedef struct { - unsigned char digest[4]; /*!< Hash bytes, big endian */ + + unsigned char digest[4]; /*!< Hash bytes, big endian */ + } XXH32_canonical_t; /*! @@ -762,7 +811,8 @@ typedef struct { * * @see @ref canonical_representation_example "Canonical Representation Example" */ -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, + XXH32_hash_t hash); /*! * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. @@ -776,105 +826,106 @@ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t * * @see @ref canonical_representation_example "Canonical Representation Example" */ -XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); - - -/*! @cond Doxygen ignores this part */ -#ifdef __has_attribute -# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) -#else -# define XXH_HAS_ATTRIBUTE(x) 0 -#endif -/*! @endcond */ - -/*! @cond Doxygen ignores this part */ -/* - * C23 __STDC_VERSION__ number hasn't been specified yet. For now - * leave as `201711L` (C17 + 1). - * TODO: Update to correct value when its been specified. - */ -#define XXH_C23_VN 201711L -/*! @endcond */ - -/*! @cond Doxygen ignores this part */ -/* C-language Attributes are added in C23. */ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) -# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) -#else -# define XXH_HAS_C_ATTRIBUTE(x) 0 -#endif -/*! @endcond */ - -/*! @cond Doxygen ignores this part */ -#if defined(__cplusplus) && defined(__has_cpp_attribute) -# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) -#else -# define XXH_HAS_CPP_ATTRIBUTE(x) 0 -#endif -/*! @endcond */ - -/*! @cond Doxygen ignores this part */ -/* - * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute - * introduced in CPP17 and C23. - * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough - * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough - */ -#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) -# define XXH_FALLTHROUGH [[fallthrough]] -#elif XXH_HAS_ATTRIBUTE(__fallthrough__) -# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) -#else -# define XXH_FALLTHROUGH /* fallthrough */ -#endif -/*! @endcond */ - -/*! @cond Doxygen ignores this part */ -/* - * Define XXH_NOESCAPE for annotated pointers in public API. - * https://clang.llvm.org/docs/AttributeReference.html#noescape - * As of writing this, only supported by clang. - */ -#if XXH_HAS_ATTRIBUTE(noescape) -# define XXH_NOESCAPE __attribute__((noescape)) -#else -# define XXH_NOESCAPE -#endif +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t +XXH32_hashFromCanonical(const XXH32_canonical_t *src); + + /*! @cond Doxygen ignores this part */ + #ifdef __has_attribute + #define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) + #else + #define XXH_HAS_ATTRIBUTE(x) 0 + #endif + /*! @endcond */ + + /*! @cond Doxygen ignores this part */ + /* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ + #define XXH_C23_VN 201711L + /*! @endcond */ + + /*! @cond Doxygen ignores this part */ + /* C-language Attributes are added in C23. */ + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && \ + defined(__has_c_attribute) + #define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) + #else + #define XXH_HAS_C_ATTRIBUTE(x) 0 + #endif + /*! @endcond */ + + /*! @cond Doxygen ignores this part */ + #if defined(__cplusplus) && defined(__has_cpp_attribute) + #define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) + #else + #define XXH_HAS_CPP_ATTRIBUTE(x) 0 + #endif + /*! @endcond */ + + /*! @cond Doxygen ignores this part */ + /* + * Define XXH_FALLTHROUGH macro for annotating switch case with the + * 'fallthrough' attribute introduced in CPP17 and C23. CPP17 : + * https://en.cppreference.com/w/cpp/language/attributes/fallthrough C23 : + * https://en.cppreference.com/w/c/language/attributes/fallthrough + */ + #if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) + #define XXH_FALLTHROUGH [[fallthrough]] + #elif XXH_HAS_ATTRIBUTE(__fallthrough__) + #define XXH_FALLTHROUGH __attribute__((__fallthrough__)) + #else + #define XXH_FALLTHROUGH /* fallthrough */ + #endif + /*! @endcond */ + + /*! @cond Doxygen ignores this part */ + /* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ + #if XXH_HAS_ATTRIBUTE(noescape) + #define XXH_NOESCAPE __attribute__((noescape)) + #else + #define XXH_NOESCAPE + #endif /*! @endcond */ - /*! * @} * @ingroup public * @{ + */ -#ifndef XXH_NO_LONG_LONG -/*-********************************************************************** -* 64-bit hash -************************************************************************/ -#if defined(XXH_DOXYGEN) /* don't include */ + #ifndef XXH_NO_LONG_LONG + /*-********************************************************************** + * 64-bit hash + ************************************************************************/ + #if defined(XXH_DOXYGEN) /* don't include */ /*! * @brief An unsigned 64-bit integer. * * Not necessarily defined to `uint64_t` but functionally equivalent. */ typedef uint64_t XXH64_hash_t; -#elif !defined (__VMS) \ - && (defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint64_t XXH64_hash_t; -#else -# include -# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL - /* LP64 ABI says uint64_t is unsigned long */ - typedef unsigned long XXH64_hash_t; -# else - /* the following type must have a width of 64-bit */ - typedef unsigned long long XXH64_hash_t; -# endif -#endif + #elif !defined(__VMS) && \ + (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 199901L) /* C99 */)) + #include +typedef uint64_t XXH64_hash_t; + #else + #include + #if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL +/* LP64 ABI says uint64_t is unsigned long */ +typedef unsigned long XXH64_hash_t; + #else +/* the following type must have a width of 64-bit */ +typedef unsigned long long XXH64_hash_t; + #endif + #endif /*! * @} @@ -882,6 +933,7 @@ typedef uint64_t XXH64_hash_t; * @defgroup XXH64_family XXH64 family * @ingroup public * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. * * @note @@ -893,7 +945,8 @@ typedef uint64_t XXH64_hash_t; /*! * @brief Calculates the 64-bit hash of @p input using xxHash64. * - * @param input The block of data to be hashed, at least @p length bytes in size. + * @param input The block of data to be hashed, at least @p length bytes in + * size. * @param length The length of @p input, in bytes. * @param seed The 64-bit seed to alter the hash's output predictably. * @@ -906,17 +959,18 @@ typedef uint64_t XXH64_hash_t; * * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void *input, + size_t length, XXH64_hash_t seed); -/******* Streaming *******/ -#ifndef XXH_NO_STREAM + /******* Streaming *******/ + #ifndef XXH_NO_STREAM /*! * @brief The opaque state struct for the XXH64 streaming API. * * @see XXH64_state_s for details. * @see @ref streaming_example "Streaming Example" */ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ /*! * @brief Allocates an @ref XXH64_state_t. @@ -928,12 +982,13 @@ typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t *XXH64_createState(void); /*! * @brief Frees an @ref XXH64_state_t. * - * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref + * XXH64_createState(). * * @return @ref XXH_OK. * @@ -941,7 +996,7 @@ XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr); /*! * @brief Copies one @ref XXH64_state_t to another. @@ -951,7 +1006,8 @@ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); * @pre * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ -XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t *dst_state, + const XXH64_state_t *src_state); /*! * @brief Resets an @ref XXH64_state_t to begin a new hash. @@ -965,17 +1021,20 @@ XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * - * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + * @note This function resets and seeds a state. Call it before @ref + * XXH64_update(). * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t *statePtr, + XXH64_hash_t seed); /*! * @brief Consumes a block of @p input to an @ref XXH64_state_t. * * @param statePtr The state struct to update. - * @param input The block of data to be hashed, at least @p length bytes in size. + * @param input The block of data to be hashed, at least @p length bytes in + * size. * @param length The length of @p input, in bytes. * * @pre @@ -992,7 +1051,9 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); +XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH_NOESCAPE XXH64_state_t *statePtr, + XXH_NOESCAPE const void *input, + size_t length); /*! * @brief Returns the calculated hash value from an @ref XXH64_state_t. @@ -1010,14 +1071,19 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); -#endif /* !XXH_NO_STREAM */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH64_digest(XXH_NOESCAPE const XXH64_state_t *statePtr); + #endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ /*! * @brief Canonical (big endian) representation of @ref XXH64_hash_t. */ -typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; +typedef struct { + + unsigned char digest[sizeof(XXH64_hash_t)]; + +} XXH64_canonical_t; /*! * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. @@ -1030,7 +1096,8 @@ typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t * * @see @ref canonical_representation_example "Canonical Representation Example" */ -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t *dst, + XXH64_hash_t hash); /*! * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. @@ -1044,9 +1111,10 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, * * @see @ref canonical_representation_example "Canonical Representation Example" */ -XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t *src); -#ifndef XXH_NO_XXH3 + #ifndef XXH_NO_XXH3 /*! * @} @@ -1054,6 +1122,7 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const * @defgroup XXH3_family XXH3 family * @ingroup public * @{ + * * XXH3 is a more recent hash algorithm featuring: * - Improved speed for both small and large inputs @@ -1085,8 +1154,9 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const * - POWER8 VSX * - s390x ZVector * This can be controlled via the @ref XXH_VECTOR macro, but it automatically - * selects the best version according to predefined macros. For the x86 family, an - * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * selects the best version according to predefined macros. For the x86 family, + * an automatic runtime dispatcher is included separately in @ref + * xxh_x86dispatch.c. * * XXH3 implementation is portable: * it has a generic C90 formulation that can be compiled on any platform, @@ -1103,13 +1173,14 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const * The API supports one-shot hashing, streaming mode, and custom secrets. */ /*-********************************************************************** -* XXH3 64-bit variant -************************************************************************/ + * XXH3 64-bit variant + ************************************************************************/ /*! * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. * - * @param input The block of data to be hashed, at least @p length bytes in size. + * @param input The block of data to be hashed, at least @p length bytes in + * size. * @param length The length of @p input, in bytes. * * @pre @@ -1120,20 +1191,22 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const * @return The calculated 64-bit XXH3 hash value. * * @note - * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however - * it may have slightly better performance due to constant propagation of the - * defaults. + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, + * however it may have slightly better performance due to constant propagation + * of the defaults. * * @see * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits(XXH_NOESCAPE const void *input, size_t length); /*! * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. * - * @param input The block of data to be hashed, at least @p length bytes in size. + * @param input The block of data to be hashed, at least @p length bytes in + * size. * @param length The length of @p input, in bytes. * @param seed The 64-bit seed to alter the hash result predictably. * @@ -1154,21 +1227,23 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input * * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed( + XXH_NOESCAPE const void *input, size_t length, XXH64_hash_t seed); -/*! - * The bare minimum size for a custom secret. - * - * @see - * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), - * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). - */ -#define XXH3_SECRET_SIZE_MIN 136 + /*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ + #define XXH3_SECRET_SIZE_MIN 136 /*! * @brief Calculates 64-bit variant of XXH3 with a custom "secret". * - * @param data The block of data to be hashed, at least @p len bytes in size. + * @param data The block of data to be hashed, at least @p len bytes in + * size. * @param len The length of @p data, in bytes. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. @@ -1180,28 +1255,29 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const vo * readable, contiguous memory. However, if @p length is `0`, @p data may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * - * It's possible to provide any blob of bytes as a "secret" to generate the hash. - * This makes it more difficult for an external actor to prepare an intentional collision. - * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). - * However, the quality of the secret impacts the dispersion of the hash algorithm. - * Therefore, the secret _must_ look like a bunch of random bytes. - * Avoid "trivial" or structured data such as repeated sequences or a text document. - * Whenever in doubt about the "randomness" of the blob of bytes, - * consider employing @ref XXH3_generateSecret() instead (see below). - * It will generate a proper high entropy secret derived from the blob of bytes. - * Another advantage of using XXH3_generateSecret() is that - * it guarantees that all bits within the initial blob of bytes - * will impact every bit of the output. - * This is not necessarily the case when using the blob of bytes directly - * because, when hashing _small_ inputs, only a portion of the secret is employed. + * It's possible to provide any blob of bytes as a "secret" to generate the + * hash. This makes it more difficult for an external actor to prepare an + * intentional collision. The main condition is that @p secretSize *must* be + * large enough (>= @ref XXH3_SECRET_SIZE_MIN). However, the quality of the + * secret impacts the dispersion of the hash algorithm. Therefore, the secret + * _must_ look like a bunch of random bytes. Avoid "trivial" or structured data + * such as repeated sequences or a text document. Whenever in doubt about the + * "randomness" of the blob of bytes, consider employing @ref + * XXH3_generateSecret() instead (see below). It will generate a proper high + * entropy secret derived from the blob of bytes. Another advantage of using + * XXH3_generateSecret() is that it guarantees that all bits within the initial + * blob of bytes will impact every bit of the output. This is not necessarily + * the case when using the blob of bytes directly because, when hashing _small_ + * inputs, only a portion of the secret is employed. * * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); - +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void *data, size_t len, + XXH_NOESCAPE const void *secret, size_t secretSize); -/******* Streaming *******/ -#ifndef XXH_NO_STREAM + /******* Streaming *******/ + #ifndef XXH_NO_STREAM /* * Streaming requires state maintenance. * This operation costs memory and CPU. @@ -1215,9 +1291,9 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const * @see XXH3_state_s for details. * @see @ref streaming_example "Streaming Example" */ -typedef struct XXH3_state_s XXH3_state_t; -XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t *XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr); /*! * @brief Copies one @ref XXH3_state_t to another. @@ -1227,7 +1303,8 @@ XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); * @pre * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ -XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t *dst_state, + XXH_NOESCAPE const XXH3_state_t *src_state); /*! * @brief Resets an @ref XXH3_state_t to begin a new hash. @@ -1241,14 +1318,16 @@ XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOE * @return @ref XXH_ERROR on failure. * * @note - * - This function resets `statePtr` and generate a secret with default parameters. + * - This function resets `statePtr` and generate a secret with default + * parameters. * - Call this function before @ref XXH3_64bits_update(). * - Digest will be equivalent to `XXH3_64bits()`. * * @see @ref streaming_example "Streaming Example" * */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t *statePtr); /*! * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. @@ -1270,7 +1349,8 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* stateP * @see @ref streaming_example "Streaming Example" * */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH64_hash_t seed); /*! * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. @@ -1296,13 +1376,16 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_ * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize); /*! * @brief Consumes a block of @p input to an @ref XXH3_state_t. * * @param statePtr The state struct to update. - * @param input The block of data to be hashed, at least @p length bytes in size. + * @param input The block of data to be hashed, at least @p length bytes in + * size. * @param length The length of @p input, in bytes. * * @pre @@ -1319,10 +1402,13 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_stat * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t *statePtr, + XXH_NOESCAPE const void *input, size_t length); /*! - * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * @brief Returns the calculated XXH3 64-bit hash value from an @ref + * XXH3_state_t. * * @param statePtr The state struct to calculate the hash from. * @@ -1332,21 +1418,21 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* stat * @return The calculated XXH3 64-bit hash value from that state. * * @note - * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, - * digest, and update again. + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can + * update, digest, and update again. * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); -#endif /* !XXH_NO_STREAM */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_digest(XXH_NOESCAPE const XXH3_state_t *statePtr); + #endif /* !XXH_NO_STREAM */ /* note : canonical representation of XXH3 is the same as XXH64 * since they both produce XXH64_hash_t values */ - /*-********************************************************************** -* XXH3 128-bit variant -************************************************************************/ + * XXH3 128-bit variant + ************************************************************************/ /*! * @brief The return value from 128-bit hashes. @@ -1355,8 +1441,10 @@ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XX * endianness. */ typedef struct { - XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ - XXH64_hash_t high64; /*!< `value >> 64` */ + + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ + } XXH128_hash_t; /*! @@ -1370,14 +1458,16 @@ typedef struct { * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead * for shorter inputs. * - * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however - * it may have slightly better performance due to constant propagation of the - * defaults. + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, + * however it may have slightly better performance due to constant propagation + * of the defaults. * - * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding + * variants * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits(XXH_NOESCAPE const void *data, size_t len); /*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. * * @param data The block of data to be hashed, at least @p length bytes in size. @@ -1397,38 +1487,42 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* dat * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed( + XXH_NOESCAPE const void *data, size_t len, XXH64_hash_t seed); /*! * @brief Calculates 128-bit variant of XXH3 with a custom "secret". * - * @param data The block of data to be hashed, at least @p len bytes in size. + * @param data The block of data to be hashed, at least @p len bytes in + * size. * @param len The length of @p data, in bytes. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * * @return The calculated 128-bit variant of XXH3 value. * - * It's possible to provide any blob of bytes as a "secret" to generate the hash. - * This makes it more difficult for an external actor to prepare an intentional collision. - * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). - * However, the quality of the secret impacts the dispersion of the hash algorithm. - * Therefore, the secret _must_ look like a bunch of random bytes. - * Avoid "trivial" or structured data such as repeated sequences or a text document. - * Whenever in doubt about the "randomness" of the blob of bytes, - * consider employing @ref XXH3_generateSecret() instead (see below). - * It will generate a proper high entropy secret derived from the blob of bytes. - * Another advantage of using XXH3_generateSecret() is that - * it guarantees that all bits within the initial blob of bytes - * will impact every bit of the output. - * This is not necessarily the case when using the blob of bytes directly - * because, when hashing _small_ inputs, only a portion of the secret is employed. + * It's possible to provide any blob of bytes as a "secret" to generate the + * hash. This makes it more difficult for an external actor to prepare an + * intentional collision. The main condition is that @p secretSize *must* be + * large enough (>= @ref XXH3_SECRET_SIZE_MIN). However, the quality of the + * secret impacts the dispersion of the hash algorithm. Therefore, the secret + * _must_ look like a bunch of random bytes. Avoid "trivial" or structured data + * such as repeated sequences or a text document. Whenever in doubt about the + * "randomness" of the blob of bytes, consider employing @ref + * XXH3_generateSecret() instead (see below). It will generate a proper high + * entropy secret derived from the blob of bytes. Another advantage of using + * XXH3_generateSecret() is that it guarantees that all bits within the initial + * blob of bytes will impact every bit of the output. This is not necessarily + * the case when using the blob of bytes directly because, when hashing _small_ + * inputs, only a portion of the secret is employed. * * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void *data, size_t len, + XXH_NOESCAPE const void *secret, size_t secretSize); -/******* Streaming *******/ -#ifndef XXH_NO_STREAM + /******* Streaming *******/ + #ifndef XXH_NO_STREAM /* * Streaming requires state maintenance. * This operation costs memory and CPU. @@ -1438,7 +1532,8 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE cons * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). * Use already declared XXH3_createState() and XXH3_freeState(). * - * All reset and streaming functions have same meaning as their 64-bit counterpart. + * All reset and streaming functions have same meaning as their 64-bit + * counterpart. */ /*! @@ -1453,13 +1548,15 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE cons * @return @ref XXH_ERROR on failure. * * @note - * - This function resets `statePtr` and generate a secret with default parameters. + * - This function resets `statePtr` and generate a secret with default + * parameters. * - Call it before @ref XXH3_128bits_update(). * - Digest will be equivalent to `XXH3_128bits()`. * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t *statePtr); /*! * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. @@ -1480,7 +1577,8 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* state * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH64_hash_t seed); /*! * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. * @@ -1503,7 +1601,9 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize); /*! * @brief Consumes a block of @p input to an @ref XXH3_state_t. @@ -1511,7 +1611,8 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_sta * Call this to incrementally consume blocks of data. * * @param statePtr The state struct to update. - * @param input The block of data to be hashed, at least @p length bytes in size. + * @param input The block of data to be hashed, at least @p length bytes in + * size. * @param length The length of @p input, in bytes. * * @pre @@ -1526,10 +1627,13 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_sta * `NULL`. In C++, this also must be *TriviallyCopyable*. * */ -XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t *statePtr, + XXH_NOESCAPE const void *input, size_t length); /*! - * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * @brief Returns the calculated XXH3 128-bit hash value from an @ref + * XXH3_state_t. * * @param statePtr The state struct to calculate the hash from. * @@ -1539,16 +1643,18 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* sta * @return The calculated XXH3 128-bit hash value from that state. * * @note - * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, - * digest, and update again. + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can + * update, digest, and update again. * */ -XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); -#endif /* !XXH_NO_STREAM */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_digest(XXH_NOESCAPE const XXH3_state_t *statePtr); + #endif /* !XXH_NO_STREAM */ /* Following helper functions make it possible to compare XXH128_hast_t values. - * Since XXH128_hash_t is a structure, this capability is not offered by the language. - * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + * Since XXH128_hash_t is a structure, this capability is not offered by the + * language. Note: For better performance, these functions can be inlined using + * XXH_INLINE_ALL */ /*! * @brief Check equality of two XXH128_hash_t values @@ -1573,15 +1679,19 @@ XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); * @return =0 if @p h128_1 == @p h128_2 * @return <0 if @p h128_1 < @p h128_2 */ -XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); - +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void *h128_1, + XXH_NOESCAPE const void *h128_2); /******* Canonical representation *******/ -typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; +typedef struct { + unsigned char digest[sizeof(XXH128_hash_t)]; + +} XXH128_canonical_t; /*! - * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * @brief Converts an @ref XXH128_hash_t to a big endian @ref + * XXH128_canonical_t. * * @param dst The @ref XXH128_canonical_t pointer to be stored to. * @param hash The @ref XXH128_hash_t to be converted. @@ -1590,7 +1700,8 @@ typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical * @p dst must not be `NULL`. * @see @ref canonical_representation_example "Canonical Representation Example" */ -XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); +XXH_PUBLIC_API void XXH128_canonicalFromHash( + XXH_NOESCAPE XXH128_canonical_t *dst, XXH128_hash_t hash); /*! * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. @@ -1603,28 +1714,27 @@ XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* ds * @return The converted hash. * @see @ref canonical_representation_example "Canonical Representation Example" */ -XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); - +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t *src); -#endif /* !XXH_NO_XXH3 */ -#endif /* XXH_NO_LONG_LONG */ + #endif /* !XXH_NO_XXH3 */ + #endif /* XXH_NO_LONG_LONG */ /*! * @} */ -#endif /* XXHASH_H_5627135585666179 */ - - +#endif /* XXHASH_H_5627135585666179 */ #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) -#define XXHASH_H_STATIC_13879238742 + #define XXHASH_H_STATIC_13879238742 /* **************************************************************************** * This section contains declarations which are not guaranteed to remain stable. * They may change in future versions, becoming incompatible with a different * version of the library. * These declarations should only be used with static linking. * Never use them in association with dynamic linking! - ***************************************************************************** */ + ***************************************************************************** +*/ /* * These definitions are only present to allow static allocation @@ -1645,16 +1755,19 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE con * @see XXH64_state_s, XXH3_state_s */ struct XXH32_state_s { - XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ - XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ - XXH32_hash_t v[4]; /*!< Accumulator lanes */ - XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ - XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ - XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ -}; /* typedef'd to XXH32_state_t */ + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref + total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as + unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ + +}; /* typedef'd to XXH32_state_t */ -#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + #ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ /*! * @internal @@ -1669,57 +1782,64 @@ struct XXH32_state_s { * @see XXH32_state_s, XXH3_state_s */ struct XXH64_state_s { - XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ - XXH64_hash_t v[4]; /*!< Accumulator lanes */ - XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ - XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ - XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ - XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ -}; /* typedef'd to XXH64_state_t */ - -#ifndef XXH_NO_XXH3 - -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ -# include -# define XXH_ALIGN(n) alignas(n) -#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ -/* In C++ alignas() is a keyword */ -# define XXH_ALIGN(n) alignas(n) -#elif defined(__GNUC__) -# define XXH_ALIGN(n) __attribute__ ((aligned(n))) -#elif defined(_MSC_VER) -# define XXH_ALIGN(n) __declspec(align(n)) -#else -# define XXH_ALIGN(n) /* disabled */ -#endif - -/* Old GCC versions only accept the attribute after the type in structures. */ -#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ - && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ - && defined(__GNUC__) -# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) -#else -# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type -#endif - -/*! - * @brief The size of the internal XXH3 buffer. - * - * This is the optimal update size for incremental hashing. - * - * @see XXH3_64b_update(), XXH3_128b_update(). - */ -#define XXH3_INTERNALBUFFER_SIZE 256 -/*! - * @internal - * @brief Default size of the secret buffer (and @ref XXH3_kSecret). - * - * This is the size used in @ref XXH3_kSecret and the seeded functions. - * - * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. - */ -#define XXH3_SECRET_DEFAULT_SIZE 192 + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as + unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ + +}; /* typedef'd to XXH64_state_t */ + + #ifndef XXH_NO_XXH3 + + #if defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L) /* >= C11 */ + #include + #define XXH_ALIGN(n) alignas(n) + #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ + /* In C++ alignas() is a keyword */ + #define XXH_ALIGN(n) alignas(n) + #elif defined(__GNUC__) + #define XXH_ALIGN(n) __attribute__((aligned(n))) + #elif defined(_MSC_VER) + #define XXH_ALIGN(n) __declspec(align(n)) + #else + #define XXH_ALIGN(n) /* disabled */ + #endif + + /* Old GCC versions only accept the attribute after the type in + * structures. */ + #if !(defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && \ + !(defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) + #define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) + #else + #define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type + #endif + + /*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ + #define XXH3_INTERNALBUFFER_SIZE 256 + + /*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ + #define XXH3_SECRET_DEFAULT_SIZE 192 /*! * @internal @@ -1744,54 +1864,60 @@ struct XXH64_state_s { * @see XXH32_state_s, XXH64_state_s */ struct XXH3_state_s { - XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); - /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ - XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); - /*!< Used to store a custom secret generated from a seed. */ - XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); - /*!< The internal buffer. @see XXH32_state_s::mem32 */ - XXH32_hash_t bufferedSize; - /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ - XXH32_hash_t useSeed; - /*!< Reserved field. Needed for padding on 64-bit. */ - size_t nbStripesSoFar; - /*!< Number or stripes processed. */ - XXH64_hash_t totalLen; - /*!< Total length hashed. 64-bit even on 32-bit targets. */ - size_t nbStripesPerBlock; - /*!< Number of stripes per block. */ - size_t secretLimit; - /*!< Size of @ref customSecret or @ref extSecret */ - XXH64_hash_t seed; - /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ - XXH64_hash_t reserved64; - /*!< Reserved field. */ - const unsigned char* extSecret; - /*!< Reference to an external secret for the _withSecret variants, NULL - * for other variants. */ - /* note: there may be some padding at the end due to alignment on 64 bytes */ -}; /* typedef'd to XXH3_state_t */ - -#undef XXH_ALIGN_MEMBER - -/*! - * @brief Initializes a stack-allocated `XXH3_state_s`. - * - * When the @ref XXH3_state_t structure is merely emplaced on stack, - * it should be initialized with XXH3_INITSTATE() or a memset() - * in case its first reset uses XXH3_NNbits_reset_withSeed(). - * This init can be omitted if the first reset uses default or _withSecret mode. - * This operation isn't necessary when the state is created with XXH3_createState(). - * Note that this doesn't prepare the state for a streaming operation, - * it's still necessary to use XXH3_NNbits_reset*() afterwards. - */ -#define XXH3_INITSTATE(XXH3_state_ptr) \ - do { \ - XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ - tmp_xxh3_state_ptr->seed = 0; \ - tmp_xxh3_state_ptr->extSecret = NULL; \ - } while(0) + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v + */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see + * XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char *extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ + +}; /* typedef'd to XXH3_state_t */ + + #undef XXH_ALIGN_MEMBER + + /*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret + * mode. This operation isn't necessary when the state is created with + * XXH3_createState(). Note that this doesn't prepare the state for a + * streaming operation, it's still necessary to use XXH3_NNbits_reset*() + * afterwards. + */ + #define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + \ + XXH3_state_t *tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + \ + } while (0) /*! * @brief Calculates the 128-bit hash of @p data using XXH3. @@ -1809,27 +1935,31 @@ struct XXH3_state_s { * * @see @ref single_shot_example "Single Shot Example" for an example. */ -XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); - +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void *data, + size_t len, XXH64_hash_t seed); /* === Experimental API === */ -/* Symbols defined below must be considered tied to a specific library version. */ +/* Symbols defined below must be considered tied to a specific library version. + */ /*! - * @brief Derive a high-entropy secret from any user-defined content, named customSeed. + * @brief Derive a high-entropy secret from any user-defined content, named + * customSeed. * - * @param secretBuffer A writable buffer for derived high-entropy secret data. - * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE. + * @param secretBuffer A writable buffer for derived high-entropy secret + * data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= + * XXH3_SECRET_DEFAULT_SIZE. * @param customSeed A user-defined content. * @param customSeedSize Size of customSeed, in bytes. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * - * The generated secret can be used in combination with `*_withSecret()` functions. - * The `_withSecret()` variants are useful to provide a higher level of protection - * than 64-bit seed, as it becomes much more difficult for an external actor to - * guess how to impact the calculation logic. + * The generated secret can be used in combination with `*_withSecret()` + * functions. The `_withSecret()` variants are useful to provide a higher level + * of protection than 64-bit seed, as it becomes much more difficult for an + * external actor to guess how to impact the calculation logic. * * The function accepts as input a custom seed of any length and any content, * and derives from it a high-entropy secret of length @p secretSize into an @@ -1839,18 +1969,20 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, siz * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() * are part of this list. They all accept a `secret` parameter - * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) - * _and_ feature very high entropy (consist of random-looking bytes). - * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can - * be employed to ensure proper quality. + * which must be large enough for implementation reasons (>= @ref + * XXH3_SECRET_SIZE_MIN) _and_ feature very high entropy (consist of + * random-looking bytes). These conditions can be a high bar to meet, so @ref + * XXH3_generateSecret() can be employed to ensure proper quality. * * @p customSeed can be anything. It can have any size, even small ones, * and its content can be anything, even "poor entropy" sources such as a bunch - * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * of zeroes. The resulting `secret` will nonetheless provide all required + * qualities. * * @pre * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN - * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined + * behavior. * * Example code: * @code{.c} @@ -1862,6 +1994,7 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, siz * // Hashes argv[2] using the entropy from argv[1]. * int main(int argc, char* argv[]) * { + * char secret[XXH3_SECRET_SIZE_MIN]; * if (argv != 3) { return 1; } * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); @@ -1873,7 +2006,9 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, siz * } * @endcode */ -XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void *secretBuffer, size_t secretSize, + XXH_NOESCAPE const void *customSeed, size_t customSeedSize); /*! * @brief Generate the same secret as the _withSeed() variants. @@ -1891,34 +2026,43 @@ XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer * #include "xxhash.h" * // Slow, seeds each time * class HashSlow { + * XXH64_hash_t seed; * public: * HashSlow(XXH64_hash_t s) : seed{s} {} * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; * } * }; * // Fast, caches the seeded secret for future uses. * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; * public: * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); * } * size_t operator()(const std::string& x) const { + * return size_t{ - * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, + *sizeof(secret)) * }; * } * }; * @endcode */ -XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed( + XXH_NOESCAPE void *secretBuffer, XXH64_hash_t seed); /*! * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. * - * @param data The block of data to be hashed, at least @p len bytes in size. + * @param data The block of data to be hashed, at least @p len bytes in + * size. * @param len The length of @p data, in bytes. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. @@ -1946,17 +2090,17 @@ XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer * On top of speed, an added benefit is that each bit in the secret * has a 50% chance to swap each bit in the output, via its impact to the seed. * - * This is not guaranteed when using the secret directly in "small data" scenarios, - * because only portions of the secret are employed for small data. + * This is not guaranteed when using the secret directly in "small data" + * scenarios, because only portions of the secret are employed for small data. */ -XXH_PUBLIC_API XXH_PUREF XXH64_hash_t -XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, - XXH_NOESCAPE const void* secret, size_t secretSize, - XXH64_hash_t seed); +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecretandSeed( + XXH_NOESCAPE const void *data, size_t len, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed); /*! * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. * - * @param input The block of data to be hashed, at least @p len bytes in size. + * @param input The block of data to be hashed, at least @p len bytes in + * size. * @param length The length of @p data, in bytes. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. @@ -1967,15 +2111,15 @@ XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, * * @see XXH3_64bits_withSecretandSeed() */ -XXH_PUBLIC_API XXH_PUREF XXH128_hash_t -XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, - XXH_NOESCAPE const void* secret, size_t secretSize, - XXH64_hash_t seed64); -#ifndef XXH_NO_STREAM +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecretandSeed( + XXH_NOESCAPE const void *input, size_t length, + XXH_NOESCAPE const void *secret, size_t secretSize, XXH64_hash_t seed64); + #ifndef XXH_NO_STREAM /*! * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. * - * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref + * XXH3_createState(). * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * @param seed64 The 64-bit seed to alter the hash result predictably. @@ -1985,14 +2129,14 @@ XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, * * @see XXH3_64bits_withSecretandSeed() */ -XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, - XXH_NOESCAPE const void* secret, size_t secretSize, - XXH64_hash_t seed64); +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecretandSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed64); /*! * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. * - * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref + * XXH3_createState(). * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * @param seed64 The 64-bit seed to alter the hash result predictably. @@ -2002,26 +2146,24 @@ XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, * * @see XXH3_64bits_withSecretandSeed() */ -XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, - XXH_NOESCAPE const void* secret, size_t secretSize, - XXH64_hash_t seed64); -#endif /* !XXH_NO_STREAM */ - -#endif /* !XXH_NO_XXH3 */ -#endif /* XXH_NO_LONG_LONG */ -#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) -# define XXH_IMPLEMENTATION -#endif +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecretandSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed64); + #endif /* !XXH_NO_STREAM */ -#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + #endif /* !XXH_NO_XXH3 */ + #endif /* XXH_NO_LONG_LONG */ + #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) + #define XXH_IMPLEMENTATION + #endif +#endif /* defined(XXH_STATIC_LINKING_ONLY) && \ + !defined(XXHASH_H_STATIC_13879238742) */ /* ======================================================================== */ /* ======================================================================== */ /* ======================================================================== */ - /*-********************************************************************** * xxHash implementation *-********************************************************************** @@ -2044,277 +2186,290 @@ XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, * which can then be linked into the final binary. ************************************************************************/ -#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ - || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) -# define XXH_IMPLEM_13a8737387 +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) || \ + defined(XXH_IMPLEMENTATION)) && \ + !defined(XXH_IMPLEM_13a8737387) + #define XXH_IMPLEM_13a8737387 + + /* ************************************* + * Tuning parameters + ***************************************/ + + /*! + * @defgroup tuning Tuning parameters + * @{ + + * + * Various macros to control xxHash's behavior. + */ + #ifdef XXH_DOXYGEN + /*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 + * compiler. + */ + #define XXH_NO_LONG_LONG + #undef XXH_NO_LONG_LONG /* don't actually */ + /*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which + * is safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated + * assembly is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers + * will eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences + * an unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's + * the only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which + * don't inline small `memcpy()` calls, and it might also be faster on + * big-endian systems which lack a native byteswap instruction. However, + * some compilers will emit literal byteshifts even if the target supports + * unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may + * cause another to read garbage data or even crash. + * + * See + * https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + * for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ + #define XXH_FORCE_MEMORY_ACCESS 0 -/* ************************************* -* Tuning parameters -***************************************/ + /*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. + * This is mostly due to heavy usage to forced inlining and constant folding + * of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. + * Speed comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies + * the options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == + * 0, and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use + * the streaming API. + */ + #define XXH_SIZE_OPT 0 -/*! - * @defgroup tuning Tuning parameters - * @{ - * - * Various macros to control xxHash's behavior. - */ -#ifdef XXH_DOXYGEN -/*! - * @brief Define this to disable 64-bit code. - * - * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. - */ -# define XXH_NO_LONG_LONG -# undef XXH_NO_LONG_LONG /* don't actually */ -/*! - * @brief Controls how unaligned memory is accessed. - * - * By default, access to unaligned memory is controlled by `memcpy()`, which is - * safe and portable. - * - * Unfortunately, on some target/compiler combinations, the generated assembly - * is sub-optimal. - * - * The below switch allow selection of a different access method - * in the search for improved performance. - * - * @par Possible options: - * - * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` - * @par - * Use `memcpy()`. Safe and portable. Note that most modern compilers will - * eliminate the function call and treat it as an unaligned access. - * - * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` - * @par - * Depends on compiler extensions and is therefore not portable. - * This method is safe _if_ your compiler supports it, - * and *generally* as fast or faster than `memcpy`. - * - * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast - * @par - * Casts directly and dereferences. This method doesn't depend on the - * compiler, but it violates the C standard as it directly dereferences an - * unaligned pointer. It can generate buggy code on targets which do not - * support unaligned memory accesses, but in some circumstances, it's the - * only known way to get the most performance. - * - * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift - * @par - * Also portable. This can generate the best code on old compilers which don't - * inline small `memcpy()` calls, and it might also be faster on big-endian - * systems which lack a native byteswap instruction. However, some compilers - * will emit literal byteshifts even if the target supports unaligned access. - * - * - * @warning - * Methods 1 and 2 rely on implementation-defined behavior. Use these with - * care, as what works on one compiler/platform/optimization level may cause - * another to read garbage data or even crash. - * - * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. - * - * Prefer these methods in priority order (0 > 3 > 1 > 2) - */ -# define XXH_FORCE_MEMORY_ACCESS 0 + /*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs + * (XXH32() and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally + * negligible, but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro + * to 0. Then the code will always use unaligned memory access. Align check + * is automatically disabled on x86, x64, ARM64, and some ARM chips which + * are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ + #define XXH_FORCE_ALIGN_CHECK 0 -/*! - * @def XXH_SIZE_OPT - * @brief Controls how much xxHash optimizes for size. - * - * xxHash, when compiled, tends to result in a rather large binary size. This - * is mostly due to heavy usage to forced inlining and constant folding of the - * @ref XXH3_family to increase performance. - * - * However, some developers prefer size over speed. This option can - * significantly reduce the size of the generated code. When using the `-Os` - * or `-Oz` options on GCC or Clang, this is defined to 1 by default, - * otherwise it is defined to 0. - * - * Most of these size optimizations can be controlled manually. - * - * This is a number from 0-2. - * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed - * comes first. - * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more - * conservative and disables hacks that increase code size. It implies the - * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, - * and @ref XXH3_NEON_LANES == 8 if they are not already defined. - * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. - * Performance may cry. For example, the single shot functions just use the - * streaming API. - */ -# define XXH_SIZE_OPT 0 + /*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all + * internal functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary + * which might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to + * performance, depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ + #define XXH_NO_INLINE_HINTS 0 + + /*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. + * This happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with + * function pointers that are *sometimes* force inline on -Og, and it is + * impossible to automatically detect this optimization level. + */ + #define XXH3_INLINE_SECRET 0 + + /*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the + * finalizer. This is generally preferable for performance, but depending on + * exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ + #define XXH32_ENDJMP 0 + + /*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use + * this. + */ + #define XXH_OLD_NAMES + #undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + /*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, + * disabling the streaming functions can improve code size significantly, + * especially with the @ref XXH3_family which tends to make constant folded + * copies of itself. + */ + #define XXH_NO_STREAM + #undef XXH_NO_STREAM /* don't actually */ + #endif /* XXH_DOXYGEN */ /*! - * @def XXH_FORCE_ALIGN_CHECK - * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() - * and XXH64() only). - * - * This is an important performance trick for architectures without decent - * unaligned memory access performance. - * - * It checks for input alignment, and when conditions are met, uses a "fast - * path" employing direct 32-bit/64-bit reads, resulting in _dramatically - * faster_ read speed. - * - * The check costs one initial branch per hash, which is generally negligible, - * but not zero. - * - * Moreover, it's not useful to generate an additional code path if memory - * access uses the same instruction for both aligned and unaligned - * addresses (e.g. x86 and aarch64). - * - * In these cases, the alignment check can be removed by setting this macro to 0. - * Then the code will always use unaligned memory access. - * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips - * which are platforms known to offer good unaligned memory accesses performance. - * - * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. - * - * This option does not affect XXH3 (only XXH32 and XXH64). + * @} */ -# define XXH_FORCE_ALIGN_CHECK 0 -/*! - * @def XXH_NO_INLINE_HINTS - * @brief When non-zero, sets all functions to `static`. - * - * By default, xxHash tries to force the compiler to inline almost all internal - * functions. - * - * This can usually improve performance due to reduced jumping and improved - * constant folding, but significantly increases the size of the binary which - * might not be favorable. - * - * Additionally, sometimes the forced inlining can be detrimental to performance, - * depending on the architecture. - * - * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the - * compiler full control on whether to inline or not. - * - * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if - * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. - */ -# define XXH_NO_INLINE_HINTS 0 - -/*! - * @def XXH3_INLINE_SECRET - * @brief Determines whether to inline the XXH3 withSecret code. - * - * When the secret size is known, the compiler can improve the performance - * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). - * - * However, if the secret size is not known, it doesn't have any benefit. This - * happens when xxHash is compiled into a global symbol. Therefore, if - * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. - * - * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers - * that are *sometimes* force inline on -Og, and it is impossible to automatically - * detect this optimization level. - */ -# define XXH3_INLINE_SECRET 0 - -/*! - * @def XXH32_ENDJMP - * @brief Whether to use a jump for `XXH32_finalize`. - * - * For performance, `XXH32_finalize` uses multiple branches in the finalizer. - * This is generally preferable for performance, - * but depending on exact architecture, a jmp may be preferable. - * - * This setting is only possibly making a difference for very small inputs. - */ -# define XXH32_ENDJMP 0 - -/*! - * @internal - * @brief Redefines old internal names. - * - * For compatibility with code that uses xxHash's internals before the names - * were changed to improve namespacing. There is no other reason to use this. - */ -# define XXH_OLD_NAMES -# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ - -/*! - * @def XXH_NO_STREAM - * @brief Disables the streaming API. - * - * When xxHash is not inlined and the streaming functions are not used, disabling - * the streaming functions can improve code size significantly, especially with - * the @ref XXH3_family which tends to make constant folded copies of itself. - */ -# define XXH_NO_STREAM -# undef XXH_NO_STREAM /* don't actually */ -#endif /* XXH_DOXYGEN */ -/*! - * @} - */ - -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ - /* prefer __packed__ structures (method 1) for GCC - * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy - * which for some reason does unaligned loads. */ -# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -#ifndef XXH_SIZE_OPT - /* default to 1 for -Os or -Oz */ -# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) -# define XXH_SIZE_OPT 1 -# else -# define XXH_SIZE_OPT 0 -# endif -#endif - -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ - /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ -# if XXH_SIZE_OPT >= 1 || \ - defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ - || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif -#endif - -#ifndef XXH_NO_INLINE_HINTS -# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ -# define XXH_NO_INLINE_HINTS 1 -# else -# define XXH_NO_INLINE_HINTS 0 -# endif -#endif - -#ifndef XXH3_INLINE_SECRET -# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ - || !defined(XXH_INLINE_ALL) -# define XXH3_INLINE_SECRET 0 -# else -# define XXH3_INLINE_SECRET 1 -# endif -#endif - -#ifndef XXH32_ENDJMP -/* generally preferable for performance */ -# define XXH32_ENDJMP 0 -#endif - -/*! - * @defgroup impl Implementation - * @{ - */ - - -/* ************************************* -* Includes & Memory related functions -***************************************/ -#if defined(XXH_NO_STREAM) -/* nothing */ -#elif defined(XXH_NO_STDLIB) + #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command \ + line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte + * shifting, so we use memcpy which for some reason does unaligned loads. */ + #if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && \ + defined(__ARM_FEATURE_UNALIGNED)) + #define XXH_FORCE_MEMORY_ACCESS 1 + #endif + #endif + + #ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ + #if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) + #define XXH_SIZE_OPT 1 + #else + #define XXH_SIZE_OPT 0 + #endif + #endif + + #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is + * available */ + #if XXH_SIZE_OPT >= 1 || defined(__i386) || defined(__x86_64__) || \ + defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) || \ + defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || \ + defined(_M_ARM) /* visual */ + #define XXH_FORCE_ALIGN_CHECK 0 + #else + #define XXH_FORCE_ALIGN_CHECK 1 + #endif + #endif + + #ifndef XXH_NO_INLINE_HINTS + #if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ + #define XXH_NO_INLINE_HINTS 1 + #else + #define XXH_NO_INLINE_HINTS 0 + #endif + #endif + + #ifndef XXH3_INLINE_SECRET + #if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) || \ + !defined(XXH_INLINE_ALL) + #define XXH3_INLINE_SECRET 0 + #else + #define XXH3_INLINE_SECRET 1 + #endif + #endif + + #ifndef XXH32_ENDJMP + /* generally preferable for performance */ + #define XXH32_ENDJMP 0 + #endif + + /*! + * @defgroup impl Implementation + * @{ + + */ + + /* ************************************* + * Includes & Memory related functions + ***************************************/ + #if defined(XXH_NO_STREAM) + /* nothing */ + #elif defined(XXH_NO_STDLIB) /* When requesting to disable any mention of stdlib, * the library loses the ability to invoked malloc / free. @@ -2325,173 +2480,212 @@ XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, * without access to dynamic allocation. */ -static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } -static void XXH_free(void* p) { (void)p; } +static XXH_CONSTF void *XXH_malloc(size_t s) { -#else + (void)s; + return NULL; -/* - * Modify the local functions below should you wish to use - * different memory routines for malloc() and free() - */ -#include +} -/*! - * @internal - * @brief Modify this function to use a different routine than malloc(). - */ -static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free(void *p) { -/*! - * @internal - * @brief Modify this function to use a different routine than free(). - */ -static void XXH_free(void* p) { free(p); } + (void)p; -#endif /* XXH_NO_STDLIB */ +} + + #else -#include + /* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ + #include /*! * @internal - * @brief Modify this function to use a different routine than memcpy(). + * @brief Modify this function to use a different routine than malloc(). */ -static void* XXH_memcpy(void* dest, const void* src, size_t size) -{ - return memcpy(dest,src,size); -} - -#include /* ULLONG_MAX */ - +static XXH_MALLOCF void *XXH_malloc(size_t s) { -/* ************************************* -* Compiler Specific Options -***************************************/ -#ifdef _MSC_VER /* Visual Studio warning fix */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif + return malloc(s); -#if XXH_NO_INLINE_HINTS /* disable inlining hints */ -# if defined(__GNUC__) || defined(__clang__) -# define XXH_FORCE_INLINE static __attribute__((unused)) -# else -# define XXH_FORCE_INLINE static -# endif -# define XXH_NO_INLINE static -/* enable inlining hints */ -#elif defined(__GNUC__) || defined(__clang__) -# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) -# define XXH_NO_INLINE static __attribute__((noinline)) -#elif defined(_MSC_VER) /* Visual Studio */ -# define XXH_FORCE_INLINE static __forceinline -# define XXH_NO_INLINE static __declspec(noinline) -#elif defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ -# define XXH_FORCE_INLINE static inline -# define XXH_NO_INLINE static -#else -# define XXH_FORCE_INLINE static -# define XXH_NO_INLINE static -#endif +} -#if XXH3_INLINE_SECRET -# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE -#else -# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE -#endif +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void *p) { + free(p); -/* ************************************* -* Debug -***************************************/ -/*! - * @ingroup tuning - * @def XXH_DEBUGLEVEL - * @brief Sets the debugging level. - * - * XXH_DEBUGLEVEL is expected to be defined externally, typically via the - * compiler's command line options. The value must be a number. - */ -#ifndef XXH_DEBUGLEVEL -# ifdef DEBUGLEVEL /* backwards compat */ -# define XXH_DEBUGLEVEL DEBUGLEVEL -# else -# define XXH_DEBUGLEVEL 0 -# endif -#endif +} -#if (XXH_DEBUGLEVEL>=1) -# include /* note: can still be disabled with NDEBUG */ -# define XXH_ASSERT(c) assert(c) -#else -# if defined(__INTEL_COMPILER) -# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) -# else -# define XXH_ASSERT(c) XXH_ASSUME(c) -# endif -#endif + #endif /* XXH_NO_STDLIB */ -/* note: use after variable declarations */ -#ifndef XXH_STATIC_ASSERT -# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ -# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) -# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ -# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) -# else -# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) -# endif -# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) -#endif + #include /*! * @internal - * @def XXH_COMPILER_GUARD(var) - * @brief Used to prevent unwanted optimizations for @p var. - * - * It uses an empty GCC inline assembly statement with a register constraint - * which forces @p var into a general purpose register (eg eax, ebx, ecx - * on x86) and marks it as modified. - * - * This is used in a few places to avoid unwanted autovectorization (e.g. - * XXH32_round()). All vectorization we want is explicit via intrinsics, - * and _usually_ isn't wanted elsewhere. - * - * We also use it to prevent unwanted constant folding for AArch64 in - * XXH3_initCustomSecret_scalar(). + * @brief Modify this function to use a different routine than memcpy(). */ -#if defined(__GNUC__) || defined(__clang__) -# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) -#else -# define XXH_COMPILER_GUARD(var) ((void)0) -#endif - -/* Specifically for NEON vectors which use the "w" constraint, on - * Clang. */ -#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) -# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) -#else -# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) -#endif - -/* ************************************* -* Basic Types -***************************************/ -#if !defined (__VMS) \ - && (defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t xxh_u8; -#else - typedef unsigned char xxh_u8; -#endif +static void *XXH_memcpy(void *dest, const void *src, size_t size) { + + return memcpy(dest, src, size); + +} + + #include /* ULLONG_MAX */ + + /* ************************************* + * Compiler Specific Options + ***************************************/ + #ifdef _MSC_VER /* Visual Studio warning fix */ + #pragma warning(disable : 4127) /* disable: C4127: conditional expression \ + is constant */ + #endif + + #if XXH_NO_INLINE_HINTS /* disable inlining hints */ + #if defined(__GNUC__) || defined(__clang__) + #define XXH_FORCE_INLINE static __attribute__((unused)) + #else + #define XXH_FORCE_INLINE static + #endif + #define XXH_NO_INLINE static + /* enable inlining hints */ + #elif defined(__GNUC__) || defined(__clang__) + #define XXH_FORCE_INLINE \ + static __inline__ __attribute__((always_inline, unused)) + #define XXH_NO_INLINE static __attribute__((noinline)) + #elif defined(_MSC_VER) /* Visual Studio */ + #define XXH_FORCE_INLINE static __forceinline + #define XXH_NO_INLINE static __declspec(noinline) + #elif defined(__cplusplus) || \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ + #define XXH_FORCE_INLINE static inline + #define XXH_NO_INLINE static + #else + #define XXH_FORCE_INLINE static + #define XXH_NO_INLINE static + #endif + + #if XXH3_INLINE_SECRET + #define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE + #else + #define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE + #endif + + /* ************************************* + * Debug + ***************************************/ + /*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ + #ifndef XXH_DEBUGLEVEL + #ifdef DEBUGLEVEL /* backwards compat */ + #define XXH_DEBUGLEVEL DEBUGLEVEL + #else + #define XXH_DEBUGLEVEL 0 + #endif + #endif + + #if (XXH_DEBUGLEVEL >= 1) + #include /* note: can still be disabled with NDEBUG */ + #define XXH_ASSERT(c) assert(c) + #else + #if defined(__INTEL_COMPILER) + #define XXH_ASSERT(c) XXH_ASSUME((unsigned char)(c)) + #else + #define XXH_ASSERT(c) XXH_ASSUME(c) + #endif + #endif + + /* note: use after variable declarations */ + #ifndef XXH_STATIC_ASSERT + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ + #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ + do { \ + \ + _Static_assert((c), m); \ + \ + } while (0) + #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ + #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ + do { \ + \ + static_assert((c), m); \ + \ + } while (0) + #else + #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ + do { \ + \ + struct xxh_sa { \ + \ + char x[(c) ? 1 : -1]; \ + \ + }; \ + \ + } while (0) + #endif + #define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c), #c) + #endif + + /*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ + #if defined(__GNUC__) || defined(__clang__) + #define XXH_COMPILER_GUARD(var) __asm__("" : "+r"(var)) + #else + #define XXH_COMPILER_GUARD(var) ((void)0) + #endif + + /* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ + #if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) + #define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w"(var)) + #else + #define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) + #endif + + /* ************************************* + * Basic Types + ***************************************/ + #if !defined(__VMS) && \ + (defined(__cplusplus) || \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) + #include +typedef uint8_t xxh_u8; + #else +typedef unsigned char xxh_u8; + #endif typedef XXH32_hash_t xxh_u32; -#ifdef XXH_OLD_NAMES -# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" -# define BYTE xxh_u8 -# define U8 xxh_u8 -# define U32 xxh_u32 -#endif + #ifdef XXH_OLD_NAMES + #warning \ + "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" + #define BYTE xxh_u8 + #define U8 xxh_u8 + #define U32 xxh_u32 + #endif /* *** Memory access *** */ @@ -2545,118 +2739,132 @@ typedef XXH32_hash_t xxh_u32; * @return The 32-bit little endian integer from the bytes at @p ptr. */ -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) -/* - * Manual byteshift. Best for old compilers which don't inline memcpy. - * We actually directly use XXH_readLE32 and XXH_readBE32. - */ -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) + /* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ + #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 2)) /* * Force direct memory access. Only works on CPU which support unaligned memory * access in hardware. */ -static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } +static xxh_u32 XXH_read32(const void *memPtr) { -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + return *(const xxh_u32 *)memPtr; -/* - * __attribute__((aligned(1))) is supported by gcc and clang. Originally the - * documentation claimed that it only increased the alignment, but actually it - * can decrease it on gcc, clang, and icc: - * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, - * https://gcc.godbolt.org/z/xYez1j67Y. - */ -#ifdef XXH_OLD_NAMES -typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; -#endif -static xxh_u32 XXH_read32(const void* ptr) -{ - typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; - return *((const xxh_unalign32*)ptr); } -#else + #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 1)) -/* - * Portable and safe solution. Generally efficient. - * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html - */ -static xxh_u32 XXH_read32(const void* memPtr) -{ - xxh_u32 val; - XXH_memcpy(&val, memPtr, sizeof(val)); - return val; -} + /* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually + * it can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ + #ifdef XXH_OLD_NAMES +typedef union { -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + xxh_u32 u32; +} __attribute__((packed)) unalign; -/* *** Endianness *** */ + #endif +static xxh_u32 XXH_read32(const void *ptr) { + + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32 *)ptr); + +} + + #else -/*! - * @ingroup tuning - * @def XXH_CPU_LITTLE_ENDIAN - * @brief Whether the target is little endian. - * - * Defined to 1 if the target is little endian, or 0 if it is big endian. - * It can be defined externally, for example on the compiler command line. - * - * If it is not defined, - * a runtime check (which is usually constant folded) is used instead. - * - * @note - * This is not necessarily defined to an integer constant. - * - * @see XXH_isLittleEndian() for the runtime check. - */ -#ifndef XXH_CPU_LITTLE_ENDIAN /* - * Try to detect endianness automatically, to avoid the nonstandard behavior - * in `XXH_isLittleEndian()` - */ -# if defined(_WIN32) /* Windows is always little endian */ \ - || defined(__LITTLE_ENDIAN__) \ - || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) -# define XXH_CPU_LITTLE_ENDIAN 1 -# elif defined(__BIG_ENDIAN__) \ - || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -# define XXH_CPU_LITTLE_ENDIAN 0 -# else + * Portable and safe solution. Generally efficient. + * see: + * https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void *memPtr) { + + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; + +} + + #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + /* *** Endianness *** */ + + /*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ + #ifndef XXH_CPU_LITTLE_ENDIAN + /* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ + #if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define XXH_CPU_LITTLE_ENDIAN 1 + #elif defined(__BIG_ENDIAN__) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define XXH_CPU_LITTLE_ENDIAN 0 + #else /*! * @internal * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. * * Most compilers will constant fold this. */ -static int XXH_isLittleEndian(void) -{ - /* - * Portable and well-defined behavior. - * Don't use static: it is detrimental to performance. - */ - const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; - return one.c[0]; -} -# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() -# endif -#endif +static int XXH_isLittleEndian(void) { + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { + xxh_u32 u; + xxh_u8 c[4]; + } one = {1}; -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + return one.c[0]; -#ifdef __has_builtin -# define XXH_HAS_BUILTIN(x) __has_builtin(x) -#else -# define XXH_HAS_BUILTIN(x) 0 -#endif +} +\ + #define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() + #endif + #endif + /* **************************************** + * Compiler-specific Functions and Macros + ******************************************/ + #define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + #ifdef __has_builtin + #define XXH_HAS_BUILTIN(x) __has_builtin(x) + #else + #define XXH_HAS_BUILTIN(x) 0 + #endif /* * C23 and future versions have standard "unreachable()". @@ -2685,142 +2893,154 @@ static int XXH_isLittleEndian(void) * doesn't work on GCC12 */ -#if XXH_HAS_BUILTIN(__builtin_unreachable) -# define XXH_UNREACHABLE() __builtin_unreachable() - -#elif defined(_MSC_VER) -# define XXH_UNREACHABLE() __assume(0) - -#else -# define XXH_UNREACHABLE() -#endif - -#if XXH_HAS_BUILTIN(__builtin_assume) -# define XXH_ASSUME(c) __builtin_assume(c) -#else -# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } -#endif - -/*! - * @internal - * @def XXH_rotl32(x,r) - * @brief 32-bit rotate left. - * - * @param x The 32-bit integer to be rotated. - * @param r The number of bits to rotate. - * @pre - * @p r > 0 && @p r < 32 - * @note - * @p x and @p r may be evaluated multiple times. - * @return The rotated result. - */ -#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ - && XXH_HAS_BUILTIN(__builtin_rotateleft64) -# define XXH_rotl32 __builtin_rotateleft32 -# define XXH_rotl64 __builtin_rotateleft64 -/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ -#elif defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) -# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) -#endif - -/*! - * @internal - * @fn xxh_u32 XXH_swap32(xxh_u32 x) - * @brief A 32-bit byteswap. - * - * @param x The 32-bit integer to byteswap. - * @return @p x, byteswapped. - */ -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -#else -static xxh_u32 XXH_swap32 (xxh_u32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -#endif - + #if XXH_HAS_BUILTIN(__builtin_unreachable) + #define XXH_UNREACHABLE() __builtin_unreachable() + + #elif defined(_MSC_VER) + #define XXH_UNREACHABLE() __assume(0) + + #else + #define XXH_UNREACHABLE() + #endif + + #if XXH_HAS_BUILTIN(__builtin_assume) + #define XXH_ASSUME(c) __builtin_assume(c) + #else + #define XXH_ASSUME(c) \ + if (!(c)) { XXH_UNREACHABLE(); } + #endif + + /*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ + #if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) && \ + XXH_HAS_BUILTIN(__builtin_rotateleft64) + #define XXH_rotl32 __builtin_rotateleft32 + #define XXH_rotl64 __builtin_rotateleft64 + /* Note: although _rotl exists for minGW (GCC under windows), performance + * seems poor */ + #elif defined(_MSC_VER) + #define XXH_rotl32(x, r) _rotl(x, r) + #define XXH_rotl64(x, r) _rotl64(x, r) + #else + #define XXH_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) + #define XXH_rotl64(x, r) (((x) << (r)) | ((x) >> (64 - (r)))) + #endif + + /*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ + #if defined(_MSC_VER) /* Visual Studio */ + #define XXH_swap32 _byteswap_ulong + #elif XXH_GCC_VERSION >= 403 + #define XXH_swap32 __builtin_bswap32 + #else +static xxh_u32 XXH_swap32(xxh_u32 x) { + + return ((x << 24) & 0xff000000) | ((x << 8) & 0x00ff0000) | + ((x >> 8) & 0x0000ff00) | ((x >> 24) & 0x000000ff); + +} + + #endif /* *************************** -* Memory reads -*****************************/ + * Memory reads + *****************************/ /*! * @internal * @brief Enum to indicate whether a pointer is aligned. */ typedef enum { - XXH_aligned, /*!< Aligned */ - XXH_unaligned /*!< Possibly unaligned */ + + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ + } XXH_alignment; -/* - * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. - * - * This is ideal for older compilers which don't inline memcpy. - */ -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + /* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ + #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) -XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) -{ - const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[0] - | ((xxh_u32)bytePtr[1] << 8) - | ((xxh_u32)bytePtr[2] << 16) - | ((xxh_u32)bytePtr[3] << 24); -} +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void *memPtr) { + + const xxh_u8 *bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] | ((xxh_u32)bytePtr[1] << 8) | ((xxh_u32)bytePtr[2] << 16) | + ((xxh_u32)bytePtr[3] << 24); -XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) -{ - const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[3] - | ((xxh_u32)bytePtr[2] << 8) - | ((xxh_u32)bytePtr[1] << 16) - | ((xxh_u32)bytePtr[0] << 24); } -#else -XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void *memPtr) { + + const xxh_u8 *bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] | ((xxh_u32)bytePtr[2] << 8) | ((xxh_u32)bytePtr[1] << 16) | + ((xxh_u32)bytePtr[0] << 24); + } -static xxh_u32 XXH_readBE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); + #else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void *ptr) { + + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + } -#endif -XXH_FORCE_INLINE xxh_u32 -XXH_readLE32_align(const void* ptr, XXH_alignment align) -{ - if (align==XXH_unaligned) { - return XXH_readLE32(ptr); - } else { - return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); - } +static xxh_u32 XXH_readBE32(const void *ptr) { + + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); + } + #endif + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32_align(const void *ptr, + XXH_alignment align) { + + if (align == XXH_unaligned) { + + return XXH_readLE32(ptr); + + } else { + + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32 *)ptr + : XXH_swap32(*(const xxh_u32 *)ptr); + + } + +} /* ************************************* -* Misc -***************************************/ + * Misc + ***************************************/ /*! @ingroup public */ -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } +XXH_PUBLIC_API unsigned XXH_versionNumber(void) { + + return XXH_VERSION_NUMBER; +} /* ******************************************************************* -* 32-bit hash functions -*********************************************************************/ + * 32-bit hash functions + *********************************************************************/ /*! * @} * @defgroup XXH32_impl XXH32 implementation @@ -2828,21 +3048,22 @@ XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } * * Details on the XXH32 implementation. * @{ + */ - /* #define instead of static const, to be used as initializers */ -#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ -#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ -#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ -#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ -#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ - -#ifdef XXH_OLD_NAMES -# define PRIME32_1 XXH_PRIME32_1 -# define PRIME32_2 XXH_PRIME32_2 -# define PRIME32_3 XXH_PRIME32_3 -# define PRIME32_4 XXH_PRIME32_4 -# define PRIME32_5 XXH_PRIME32_5 -#endif +/* #define instead of static const, to be used as initializers */ + #define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ + #define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ + #define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ + #define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ + #define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + + #ifdef XXH_OLD_NAMES + #define PRIME32_1 XXH_PRIME32_1 + #define PRIME32_2 XXH_PRIME32_2 + #define PRIME32_3 XXH_PRIME32_3 + #define PRIME32_4 XXH_PRIME32_4 + #define PRIME32_5 XXH_PRIME32_5 + #endif /*! * @internal @@ -2855,51 +3076,54 @@ XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } * @param input The stripe of input to mix. * @return The mixed accumulator lane. */ -static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) -{ - acc += input * XXH_PRIME32_2; - acc = XXH_rotl32(acc, 13); - acc *= XXH_PRIME32_1; -#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) - /* - * UGLY HACK: - * A compiler fence is the only thing that prevents GCC and Clang from - * autovectorizing the XXH32 loop (pragmas and attributes don't work for some - * reason) without globally disabling SSE4.1. - * - * The reason we want to avoid vectorization is because despite working on - * 4 integers at a time, there are multiple factors slowing XXH32 down on - * SSE4: - * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on - * newer chips!) making it slightly slower to multiply four integers at - * once compared to four integers independently. Even when pmulld was - * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE - * just to multiply unless doing a long operation. - * - * - Four instructions are required to rotate, - * movqda tmp, v // not required with VEX encoding - * pslld tmp, 13 // tmp <<= 13 - * psrld v, 19 // x >>= 19 - * por v, tmp // x |= tmp - * compared to one for scalar: - * roll v, 13 // reliably fast across the board - * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason - * - * - Instruction level parallelism is actually more beneficial here because - * the SIMD actually serializes this operation: While v1 is rotating, v2 - * can load data, while v3 can multiply. SSE forces them to operate - * together. - * - * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing - * the loop. NEON is only faster on the A53, and with the newer cores, it is less - * than half the speed. - * - * Additionally, this is used on WASM SIMD128 because it JITs to the same - * SIMD instructions and has the same issue. - */ - XXH_COMPILER_GUARD(acc); -#endif - return acc; +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { + + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; + #if (defined(__SSE4_1__) || defined(__aarch64__) || \ + defined(__wasm_simd128__)) && \ + !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in + * vectorizing the loop. NEON is only faster on the A53, and with the newer + * cores, it is less than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); + #endif + return acc; + } /*! @@ -2912,17 +3136,18 @@ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) * @param hash The hash to avalanche. * @return The avalanched hash. */ -static xxh_u32 XXH32_avalanche(xxh_u32 hash) -{ - hash ^= hash >> 15; - hash *= XXH_PRIME32_2; - hash ^= hash >> 13; - hash *= XXH_PRIME32_3; - hash ^= hash >> 16; - return hash; +static xxh_u32 XXH32_avalanche(xxh_u32 hash) { + + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; + } -#define XXH_get32bits(p) XXH_readLE32_align(p, align) + #define XXH_get32bits(p) XXH_readLE32_align(p, align) /*! * @internal @@ -2939,86 +3164,122 @@ static xxh_u32 XXH32_avalanche(xxh_u32 hash) * @return The finalized hash. * @see XXH64_finalize(). */ -static XXH_PUREF xxh_u32 -XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) -{ -#define XXH_PROCESS1 do { \ - hash += (*ptr++) * XXH_PRIME32_5; \ - hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ -} while (0) - -#define XXH_PROCESS4 do { \ - hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ - ptr += 4; \ - hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ -} while (0) - - if (ptr==NULL) XXH_ASSERT(len == 0); - - /* Compact rerolled version; generally faster */ - if (!XXH32_ENDJMP) { - len &= 15; - while (len >= 4) { - XXH_PROCESS4; - len -= 4; - } - while (len > 0) { - XXH_PROCESS1; - --len; - } +static XXH_PUREF xxh_u32 XXH32_finalize(xxh_u32 hash, const xxh_u8 *ptr, + size_t len, XXH_alignment align) { +\ + #define XXH_PROCESS1 \ + do { \ + \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ + \ + } while (0) + + #define XXH_PROCESS4 \ + do { \ + \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ + \ + } while (0) + + if (ptr == NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + + len &= 15; + while (len >= 4) { + + XXH_PROCESS4; + len -= 4; + + } + + while (len > 0) { + + XXH_PROCESS1; + --len; + + } + + return XXH32_avalanche(hash); + + } else { + + switch (len & 15) /* or switch(bEnd - p) */ { + + case 12: + XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: + XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: + XXH_PROCESS4; return XXH32_avalanche(hash); - } else { - switch(len&15) /* or switch(bEnd - p) */ { - case 12: XXH_PROCESS4; - XXH_FALLTHROUGH; /* fallthrough */ - case 8: XXH_PROCESS4; - XXH_FALLTHROUGH; /* fallthrough */ - case 4: XXH_PROCESS4; - return XXH32_avalanche(hash); - - case 13: XXH_PROCESS4; - XXH_FALLTHROUGH; /* fallthrough */ - case 9: XXH_PROCESS4; - XXH_FALLTHROUGH; /* fallthrough */ - case 5: XXH_PROCESS4; - XXH_PROCESS1; - return XXH32_avalanche(hash); - - case 14: XXH_PROCESS4; - XXH_FALLTHROUGH; /* fallthrough */ - case 10: XXH_PROCESS4; - XXH_FALLTHROUGH; /* fallthrough */ - case 6: XXH_PROCESS4; - XXH_PROCESS1; - XXH_PROCESS1; - return XXH32_avalanche(hash); - - case 15: XXH_PROCESS4; - XXH_FALLTHROUGH; /* fallthrough */ - case 11: XXH_PROCESS4; - XXH_FALLTHROUGH; /* fallthrough */ - case 7: XXH_PROCESS4; - XXH_FALLTHROUGH; /* fallthrough */ - case 3: XXH_PROCESS1; - XXH_FALLTHROUGH; /* fallthrough */ - case 2: XXH_PROCESS1; - XXH_FALLTHROUGH; /* fallthrough */ - case 1: XXH_PROCESS1; - XXH_FALLTHROUGH; /* fallthrough */ - case 0: return XXH32_avalanche(hash); - } - XXH_ASSERT(0); - return hash; /* reaching this point is deemed impossible */ + + case 13: + XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: + XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: + XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: + XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: + XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: + XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: + XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: + XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: + XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: + XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: + XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: + XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: + return XXH32_avalanche(hash); + } + + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + + } + } -#ifdef XXH_OLD_NAMES -# define PROCESS1 XXH_PROCESS1 -# define PROCESS4 XXH_PROCESS4 -#else -# undef XXH_PROCESS1 -# undef XXH_PROCESS4 -#endif + #ifdef XXH_OLD_NAMES + #define PROCESS1 XXH_PROCESS1 + #define PROCESS4 XXH_PROCESS4 + #else + #undef XXH_PROCESS1 + #undef XXH_PROCESS4 + #endif /*! * @internal @@ -3028,372 +3289,459 @@ XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) * @param align Whether @p input is aligned. * @return The calculated hash. */ -XXH_FORCE_INLINE XXH_PUREF xxh_u32 -XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) -{ - xxh_u32 h32; - - if (input==NULL) XXH_ASSERT(len == 0); - - if (len>=16) { - const xxh_u8* const bEnd = input + len; - const xxh_u8* const limit = bEnd - 15; - xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; - xxh_u32 v2 = seed + XXH_PRIME32_2; - xxh_u32 v3 = seed + 0; - xxh_u32 v4 = seed - XXH_PRIME32_1; - - do { - v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; - v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; - v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; - v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; - } while (input < limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) - + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + XXH_PRIME32_5; - } +XXH_FORCE_INLINE XXH_PUREF xxh_u32 XXH32_endian_align(const xxh_u8 *input, + size_t len, xxh_u32 seed, + XXH_alignment align) { - h32 += (xxh_u32)len; + xxh_u32 h32; - return XXH32_finalize(h32, input, len&15, align); -} + if (input == NULL) XXH_ASSERT(len == 0); -/*! @ingroup XXH32_family */ -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) -{ -#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, (const xxh_u8*)input, len); - return XXH32_digest(&state); -#else - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); - } } - - return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); -#endif -} + if (len >= 16) { + const xxh_u8 *const bEnd = input + len; + const xxh_u8 *const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + do { -/******* Hash streaming *******/ -#ifndef XXH_NO_STREAM -/*! @ingroup XXH32_family */ -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -/*! @ingroup XXH32_family */ -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} + v1 = XXH32_round(v1, XXH_get32bits(input)); + input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); + input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); + input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); + input += 4; -/*! @ingroup XXH32_family */ -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) -{ - XXH_memcpy(dstState, srcState, sizeof(*dstState)); -} + } while (input < limit); -/*! @ingroup XXH32_family */ -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) -{ - XXH_ASSERT(statePtr != NULL); - memset(statePtr, 0, sizeof(*statePtr)); - statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; - statePtr->v[1] = seed + XXH_PRIME32_2; - statePtr->v[2] = seed + 0; - statePtr->v[3] = seed - XXH_PRIME32_1; - return XXH_OK; -} + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + + XXH_rotl32(v4, 18); + } else { -/*! @ingroup XXH32_family */ -XXH_PUBLIC_API XXH_errorcode -XXH32_update(XXH32_state_t* state, const void* input, size_t len) -{ - if (input==NULL) { - XXH_ASSERT(len == 0); - return XXH_OK; - } + h32 = seed + XXH_PRIME32_5; - { const xxh_u8* p = (const xxh_u8*)input; - const xxh_u8* const bEnd = p + len; + } - state->total_len_32 += (XXH32_hash_t)len; - state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + h32 += (xxh_u32)len; - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); - state->memsize += (XXH32_hash_t)len; - return XXH_OK; - } + return XXH32_finalize(h32, input, len & 15, align); - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const xxh_u32* p32 = state->mem32; - state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; - state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; - state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; - state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); - } - p += 16-state->memsize; - state->memsize = 0; - } +} - if (p <= bEnd-16) { - const xxh_u8* const limit = bEnd - 16; +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, + XXH32_hash_t seed) { + + #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small + * inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8 *)input, len); + return XXH32_digest(&state); + #else + if (XXH_FORCE_ALIGN_CHECK) { + + if ((((size_t)input) & 3) == + 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8 *)input, len, seed, XXH_aligned); - do { - state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; - state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; - state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; - state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; - } while (p<=limit); + } - } + } - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - } + return XXH32_endian_align((const xxh_u8 *)input, len, seed, XXH_unaligned); + #endif - return XXH_OK; } + /******* Hash streaming *******/ + #ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t *XXH32_createState(void) { + + return (XXH32_state_t *)XXH_malloc(sizeof(XXH32_state_t)); + +} /*! @ingroup XXH32_family */ -XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) -{ - xxh_u32 h32; +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr) { + + XXH_free(statePtr); + return XXH_OK; + +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t *dstState, + const XXH32_state_t *srcState) { + + XXH_memcpy(dstState, srcState, sizeof(*dstState)); + +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, + XXH32_hash_t seed) { + + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; + +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, + const void *input, size_t len) { + + if (input == NULL) { + + XXH_ASSERT(len == 0); + return XXH_OK; + + } + + { + + const xxh_u8 *p = (const xxh_u8 *)input; + const xxh_u8 *const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= + (XXH32_hash_t)((len >= 16) | (state->total_len_32 >= 16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8 *)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8 *)(state->mem32) + state->memsize, input, + 16 - state->memsize); + { + + const xxh_u32 *p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); + p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); + p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); + p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + + } + + p += 16 - state->memsize; + state->memsize = 0; - if (state->large_len) { - h32 = XXH_rotl32(state->v[0], 1) - + XXH_rotl32(state->v[1], 7) - + XXH_rotl32(state->v[2], 12) - + XXH_rotl32(state->v[3], 18); - } else { - h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; } - h32 += state->total_len_32; + if (p <= bEnd - 16) { + + const xxh_u8 *const limit = bEnd - 16; + + do { + + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); + p += 4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); + p += 4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); + p += 4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); + p += 4; + + } while (p <= limit); + + } + + if (p < bEnd) { + + XXH_memcpy(state->mem32, p, (size_t)(bEnd - p)); + state->memsize = (unsigned)(bEnd - p); + + } + + } + + return XXH_OK; + +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *state) { + + xxh_u32 h32; + + if (state->large_len) { + + h32 = XXH_rotl32(state->v[0], 1) + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + XXH_rotl32(state->v[3], 18); + + } else { + + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8 *)state->mem32, state->memsize, + XXH_aligned); - return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); } -#endif /* !XXH_NO_STREAM */ + + #endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ /*! @ingroup XXH32_family */ -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - XXH_memcpy(dst, &hash, sizeof(*dst)); +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, + XXH32_hash_t hash) { + + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); + } + /*! @ingroup XXH32_family */ -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} +XXH_PUBLIC_API XXH32_hash_t +XXH32_hashFromCanonical(const XXH32_canonical_t *src) { + + return XXH_readBE32(src); +} -#ifndef XXH_NO_LONG_LONG + #ifndef XXH_NO_LONG_LONG /* ******************************************************************* -* 64-bit hash functions -*********************************************************************/ + * 64-bit hash functions + *********************************************************************/ /*! * @} * @ingroup impl * @{ + */ /******* Memory access *******/ typedef XXH64_hash_t xxh_u64; -#ifdef XXH_OLD_NAMES -# define U64 xxh_u64 -#endif + #ifdef XXH_OLD_NAMES + #define U64 xxh_u64 + #endif -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) -/* - * Manual byteshift. Best for old compilers which don't inline memcpy. - * We actually directly use XXH_readLE64 and XXH_readBE64. - */ -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) + /* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ + #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware */ +static xxh_u64 XXH_read64(const void *memPtr) { + + return *(const xxh_u64 *)memPtr; -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static xxh_u64 XXH_read64(const void* memPtr) -{ - return *(const xxh_u64*) memPtr; } -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 1)) + + /* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally + * the documentation claimed that it only increased the alignment, but + * actually it can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ + #ifdef XXH_OLD_NAMES +typedef union { + + xxh_u32 u32; + xxh_u64 u64; + +} __attribute__((packed)) unalign64; + + #endif +static xxh_u64 XXH_read64(const void *ptr) { + + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64 *)ptr); -/* - * __attribute__((aligned(1))) is supported by gcc and clang. Originally the - * documentation claimed that it only increased the alignment, but actually it - * can decrease it on gcc, clang, and icc: - * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, - * https://gcc.godbolt.org/z/xYez1j67Y. - */ -#ifdef XXH_OLD_NAMES -typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; -#endif -static xxh_u64 XXH_read64(const void* ptr) -{ - typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; - return *((const xxh_unalign64*)ptr); } -#else + #else /* * Portable and safe solution. Generally efficient. - * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + * see: + * https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html */ -static xxh_u64 XXH_read64(const void* memPtr) -{ - xxh_u64 val; - XXH_memcpy(&val, memPtr, sizeof(val)); - return val; +static xxh_u64 XXH_read64(const void *memPtr) { + + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; + } -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + #if defined(_MSC_VER) /* Visual Studio */ + #define XXH_swap64 _byteswap_uint64 + #elif XXH_GCC_VERSION >= 403 + #define XXH_swap64 __builtin_bswap64 + #else +static xxh_u64 XXH_swap64(xxh_u64 x) { + + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap64 _byteswap_uint64 -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap64 __builtin_bswap64 -#else -static xxh_u64 XXH_swap64(xxh_u64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); } -#endif + #endif -/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + /* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ + #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void *memPtr) { + + const xxh_u8 *bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] | ((xxh_u64)bytePtr[1] << 8) | ((xxh_u64)bytePtr[2] << 16) | + ((xxh_u64)bytePtr[3] << 24) | ((xxh_u64)bytePtr[4] << 32) | + ((xxh_u64)bytePtr[5] << 40) | ((xxh_u64)bytePtr[6] << 48) | + ((xxh_u64)bytePtr[7] << 56); -XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) -{ - const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[0] - | ((xxh_u64)bytePtr[1] << 8) - | ((xxh_u64)bytePtr[2] << 16) - | ((xxh_u64)bytePtr[3] << 24) - | ((xxh_u64)bytePtr[4] << 32) - | ((xxh_u64)bytePtr[5] << 40) - | ((xxh_u64)bytePtr[6] << 48) - | ((xxh_u64)bytePtr[7] << 56); -} - -XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) -{ - const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; - return bytePtr[7] - | ((xxh_u64)bytePtr[6] << 8) - | ((xxh_u64)bytePtr[5] << 16) - | ((xxh_u64)bytePtr[4] << 24) - | ((xxh_u64)bytePtr[3] << 32) - | ((xxh_u64)bytePtr[2] << 40) - | ((xxh_u64)bytePtr[1] << 48) - | ((xxh_u64)bytePtr[0] << 56); -} - -#else -XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); } -static xxh_u64 XXH_readBE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void *memPtr) { + + const xxh_u8 *bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] | ((xxh_u64)bytePtr[6] << 8) | ((xxh_u64)bytePtr[5] << 16) | + ((xxh_u64)bytePtr[4] << 24) | ((xxh_u64)bytePtr[3] << 32) | + ((xxh_u64)bytePtr[2] << 40) | ((xxh_u64)bytePtr[1] << 48) | + ((xxh_u64)bytePtr[0] << 56); + } -#endif -XXH_FORCE_INLINE xxh_u64 -XXH_readLE64_align(const void* ptr, XXH_alignment align) -{ - if (align==XXH_unaligned) - return XXH_readLE64(ptr); - else - return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); + #else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void *ptr) { + + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + } +static xxh_u64 XXH_readBE64(const void *ptr) { -/******* xxh64 *******/ -/*! - * @} - * @defgroup XXH64_impl XXH64 implementation - * @ingroup impl - * - * Details on the XXH64 implementation. - * @{ - */ -/* #define rather that static const, to be used as initializers */ -#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ -#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ -#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ -#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ -#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ - -#ifdef XXH_OLD_NAMES -# define PRIME64_1 XXH_PRIME64_1 -# define PRIME64_2 XXH_PRIME64_2 -# define PRIME64_3 XXH_PRIME64_3 -# define PRIME64_4 XXH_PRIME64_4 -# define PRIME64_5 XXH_PRIME64_5 -#endif + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); + +} + + #endif + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64_align(const void *ptr, + XXH_alignment align) { + + if (align == XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64 *)ptr + : XXH_swap64(*(const xxh_u64 *)ptr); + +} + + /******* xxh64 *******/ + /*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + + */ + /* #define rather that static const, to be used as initializers */ + #define XXH_PRIME64_1 \ + 0x9E3779B185EBCA87ULL /*!< \ + 0b1001111000110111011110011011000110000101111010111100101010000111 \ + */ + #define XXH_PRIME64_2 \ + 0xC2B2AE3D27D4EB4FULL /*!< \ + 0b1100001010110010101011100011110100100111110101001110101101001111 \ + */ + #define XXH_PRIME64_3 \ + 0x165667B19E3779F9ULL /*!< \ + 0b0001011001010110011001111011000110011110001101110111100111111001 \ + */ + #define XXH_PRIME64_4 \ + 0x85EBCA77C2B2AE63ULL /*!< \ + 0b1000010111101011110010100111011111000010101100101010111001100011 \ + */ + #define XXH_PRIME64_5 \ + 0x27D4EB2F165667C5ULL /*!< \ + 0b0010011111010100111010110010111100010110010101100110011111000101 \ + */ + + #ifdef XXH_OLD_NAMES + #define PRIME64_1 XXH_PRIME64_1 + #define PRIME64_2 XXH_PRIME64_2 + #define PRIME64_3 XXH_PRIME64_3 + #define PRIME64_4 XXH_PRIME64_4 + #define PRIME64_5 XXH_PRIME64_5 + #endif /*! @copydoc XXH32_round */ -static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) -{ - acc += input * XXH_PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= XXH_PRIME64_1; - return acc; +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) { + + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; + } -static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; - return acc; +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) { + + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; + } /*! @copydoc XXH32_avalanche */ -static xxh_u64 XXH64_avalanche(xxh_u64 hash) -{ - hash ^= hash >> 33; - hash *= XXH_PRIME64_2; - hash ^= hash >> 29; - hash *= XXH_PRIME64_3; - hash ^= hash >> 32; - return hash; -} +static xxh_u64 XXH64_avalanche(xxh_u64 hash) { + + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} -#define XXH_get64bits(p) XXH_readLE64_align(p, align) + #define XXH_get64bits(p) XXH_readLE64_align(p, align) /*! * @internal @@ -3410,41 +3758,51 @@ static xxh_u64 XXH64_avalanche(xxh_u64 hash) * @return The finalized hash * @see XXH32_finalize(). */ -static XXH_PUREF xxh_u64 -XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) -{ - if (ptr==NULL) XXH_ASSERT(len == 0); - len &= 31; - while (len >= 8) { - xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); - ptr += 8; - hash ^= k1; - hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; - len -= 8; - } - if (len >= 4) { - hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; - ptr += 4; - hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; - len -= 4; - } - while (len > 0) { - hash ^= (*ptr++) * XXH_PRIME64_5; - hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; - --len; - } - return XXH64_avalanche(hash); +static XXH_PUREF xxh_u64 XXH64_finalize(xxh_u64 hash, const xxh_u8 *ptr, + size_t len, XXH_alignment align) { + + if (ptr == NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + + } + + if (len >= 4) { + + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + + } + + while (len > 0) { + + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + + } + + return XXH64_avalanche(hash); + } -#ifdef XXH_OLD_NAMES -# define PROCESS1_64 XXH_PROCESS1_64 -# define PROCESS4_64 XXH_PROCESS4_64 -# define PROCESS8_64 XXH_PROCESS8_64 -#else -# undef XXH_PROCESS1_64 -# undef XXH_PROCESS4_64 -# undef XXH_PROCESS8_64 -#endif + #ifdef XXH_OLD_NAMES + #define PROCESS1_64 XXH_PROCESS1_64 + #define PROCESS4_64 XXH_PROCESS4_64 + #define PROCESS8_64 XXH_PROCESS8_64 + #else + #undef XXH_PROCESS1_64 + #undef XXH_PROCESS4_64 + #undef XXH_PROCESS8_64 + #endif /*! * @internal @@ -3454,349 +3812,416 @@ XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) * @param align Whether @p input is aligned. * @return The calculated hash. */ -XXH_FORCE_INLINE XXH_PUREF xxh_u64 -XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) -{ - xxh_u64 h64; - if (input==NULL) XXH_ASSERT(len == 0); - - if (len>=32) { - const xxh_u8* const bEnd = input + len; - const xxh_u8* const limit = bEnd - 31; - xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; - xxh_u64 v2 = seed + XXH_PRIME64_2; - xxh_u64 v3 = seed + 0; - xxh_u64 v4 = seed - XXH_PRIME64_1; - - do { - v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; - v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; - v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; - v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; - } while (input= 32) { - return XXH64_finalize(h64, input, len, align); -} + const xxh_u8 *const bEnd = input + len; + const xxh_u8 *const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + + v1 = XXH64_round(v1, XXH_get64bits(input)); + input += 8; + v2 = XXH64_round(v2, XXH_get64bits(input)); + input += 8; + v3 = XXH64_round(v3, XXH_get64bits(input)); + input += 8; + v4 = XXH64_round(v4, XXH_get64bits(input)); + input += 8; + + } while (input < limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + + h64 = seed + XXH_PRIME64_5; + + } + + h64 += (xxh_u64)len; + return XXH64_finalize(h64, input, len, align); + +} /*! @ingroup XXH64_family */ -XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) -{ -#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_state_t state; - XXH64_reset(&state, seed); - XXH64_update(&state, (const xxh_u8*)input, len); - return XXH64_digest(&state); -#else - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); - } } - - return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +XXH_PUBLIC_API XXH64_hash_t XXH64(XXH_NOESCAPE const void *input, size_t len, + XXH64_hash_t seed) { + + #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small + * inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8 *)input, len); + return XXH64_digest(&state); + #else + if (XXH_FORCE_ALIGN_CHECK) { + + if ((((size_t)input) & 7) == + 0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8 *)input, len, seed, XXH_aligned); + + } + + } + + return XXH64_endian_align((const xxh_u8 *)input, len, seed, XXH_unaligned); + + #endif -#endif } -/******* Hash Streaming *******/ -#ifndef XXH_NO_STREAM + /******* Hash Streaming *******/ + #ifndef XXH_NO_STREAM /*! @ingroup XXH64_family*/ -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) -{ - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +XXH_PUBLIC_API XXH64_state_t *XXH64_createState(void) { + + return (XXH64_state_t *)XXH_malloc(sizeof(XXH64_state_t)); + } + /*! @ingroup XXH64_family */ -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr) { + + XXH_free(statePtr); + return XXH_OK; + } /*! @ingroup XXH64_family */ -XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) -{ - XXH_memcpy(dstState, srcState, sizeof(*dstState)); +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t *dstState, + const XXH64_state_t *srcState) { + + XXH_memcpy(dstState, srcState, sizeof(*dstState)); + } /*! @ingroup XXH64_family */ -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) -{ - XXH_ASSERT(statePtr != NULL); - memset(statePtr, 0, sizeof(*statePtr)); - statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; - statePtr->v[1] = seed + XXH_PRIME64_2; - statePtr->v[2] = seed + 0; - statePtr->v[3] = seed - XXH_PRIME64_1; - return XXH_OK; +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t *statePtr, + XXH64_hash_t seed) { + + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; + } /*! @ingroup XXH64_family */ -XXH_PUBLIC_API XXH_errorcode -XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) -{ - if (input==NULL) { - XXH_ASSERT(len == 0); - return XXH_OK; - } +XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH_NOESCAPE XXH64_state_t *state, + XXH_NOESCAPE const void *input, + size_t len) { - { const xxh_u8* p = (const xxh_u8*)input; - const xxh_u8* const bEnd = p + len; + if (input == NULL) { - state->total_len += len; + XXH_ASSERT(len == 0); + return XXH_OK; - if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); - state->memsize += (xxh_u32)len; - return XXH_OK; - } + } - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); - state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); - state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); - state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); - p += 32 - state->memsize; - state->memsize = 0; - } + { - if (p+32 <= bEnd) { - const xxh_u8* const limit = bEnd - 32; + const xxh_u8 *p = (const xxh_u8 *)input; + const xxh_u8 *const bEnd = p + len; - do { - state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; - state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; - state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; - state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; - } while (p<=limit); + state->total_len += len; - } + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8 *)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } } - return XXH_OK; -} + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8 *)state->mem64) + state->memsize, input, + 32 - state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64 + 0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64 + 1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64 + 2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64 + 3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p + 32 <= bEnd) { + + const xxh_u8 *const limit = bEnd - 32; + + do { + + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); + p += 8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); + p += 8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); + p += 8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); + p += 8; + + } while (p <= limit); -/*! @ingroup XXH64_family */ -XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) -{ - xxh_u64 h64; - - if (state->total_len >= 32) { - h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); - h64 = XXH64_mergeRound(h64, state->v[0]); - h64 = XXH64_mergeRound(h64, state->v[1]); - h64 = XXH64_mergeRound(h64, state->v[2]); - h64 = XXH64_mergeRound(h64, state->v[3]); - } else { - h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; } - h64 += (xxh_u64) state->total_len; + if (p < bEnd) { - return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); -} -#endif /* !XXH_NO_STREAM */ + XXH_memcpy(state->mem64, p, (size_t)(bEnd - p)); + state->memsize = (unsigned)(bEnd - p); -/******* Canonical representation *******/ + } + + } + + return XXH_OK; -/*! @ingroup XXH64_family */ -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - XXH_memcpy(dst, &hash, sizeof(*dst)); } /*! @ingroup XXH64_family */ -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); +XXH_PUBLIC_API XXH64_hash_t +XXH64_digest(XXH_NOESCAPE const XXH64_state_t *state) { + + xxh_u64 h64; + + if (state->total_len >= 32) { + + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + + } else { + + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + + } + + h64 += (xxh_u64)state->total_len; + + return XXH64_finalize(h64, (const xxh_u8 *)state->mem64, + (size_t)state->total_len, XXH_aligned); + } -#ifndef XXH_NO_XXH3 + #endif /* !XXH_NO_STREAM */ -/* ********************************************************************* -* XXH3 -* New generation hash designed for speed on small keys and vectorization -************************************************************************ */ -/*! - * @} - * @defgroup XXH3_impl XXH3 implementation - * @ingroup impl - * @{ - */ +/******* Canonical representation *******/ -/* === Compiler specifics === */ +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t *dst, + XXH64_hash_t hash) { -#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ -# define XXH_RESTRICT /* disable */ -#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ -# define XXH_RESTRICT restrict -#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ - || (defined (__clang__)) \ - || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ - || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) -/* - * There are a LOT more compilers that recognize __restrict but this - * covers the major ones. - */ -# define XXH_RESTRICT __restrict -#else -# define XXH_RESTRICT /* disable */ -#endif + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); -#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ - || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ - || defined(__clang__) -# define XXH_likely(x) __builtin_expect(x, 1) -# define XXH_unlikely(x) __builtin_expect(x, 0) -#else -# define XXH_likely(x) (x) -# define XXH_unlikely(x) (x) -#endif +} -#ifndef XXH_HAS_INCLUDE -# ifdef __has_include -/* - * Not defined as XXH_HAS_INCLUDE(x) (function-like) because - * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) - */ -# define XXH_HAS_INCLUDE __has_include -# else -# define XXH_HAS_INCLUDE(x) 0 -# endif -#endif +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t *src) { -#if defined(__GNUC__) || defined(__clang__) -# if defined(__ARM_FEATURE_SVE) -# include -# endif -# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ - || (defined(_M_ARM) && _M_ARM >= 7) \ - || defined(_M_ARM64) || defined(_M_ARM64EC) \ - || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ -# define inline __inline__ /* circumvent a clang bug */ -# include -# undef inline -# elif defined(__AVX2__) -# include -# elif defined(__SSE2__) -# include -# endif -#endif + return XXH_readBE64(src); -#if defined(_MSC_VER) -# include -#endif +} -/* - * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while - * remaining a true 64-bit/128-bit hash function. - * - * This is done by prioritizing a subset of 64-bit operations that can be - * emulated without too many steps on the average 32-bit machine. - * - * For example, these two lines seem similar, and run equally fast on 64-bit: - * - * xxh_u64 x; - * x ^= (x >> 47); // good - * x ^= (x >> 13); // bad - * - * However, to a 32-bit machine, there is a major difference. - * - * x ^= (x >> 47) looks like this: - * - * x.lo ^= (x.hi >> (47 - 32)); - * - * while x ^= (x >> 13) looks like this: - * - * // note: funnel shifts are not usually cheap. - * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); - * x.hi ^= (x.hi >> 13); - * - * The first one is significantly faster than the second, simply because the - * shift is larger than 32. This means: - * - All the bits we need are in the upper 32 bits, so we can ignore the lower - * 32 bits in the shift. - * - The shift result will always fit in the lower 32 bits, and therefore, - * we can ignore the upper 32 bits in the xor. - * - * Thanks to this optimization, XXH3 only requires these features to be efficient: - * - * - Usable unaligned access - * - A 32-bit or 64-bit ALU - * - If 32-bit, a decent ADC instruction - * - A 32 or 64-bit multiply with a 64-bit result - * - For the 128-bit variant, a decent byteswap helps short inputs. - * - * The first two are already required by XXH32, and almost all 32-bit and 64-bit - * platforms which can run XXH32 can run XXH3 efficiently. - * - * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one - * notable exception. - * - * First of all, Thumb-1 lacks support for the UMULL instruction which - * performs the important long multiply. This means numerous __aeabi_lmul - * calls. - * - * Second of all, the 8 functional registers are just not enough. - * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need - * Lo registers, and this shuffling results in thousands more MOVs than A32. - * - * A32 and T32 don't have this limitation. They can access all 14 registers, - * do a 32->64 multiply with UMULL, and the flexible operand allowing free - * shifts is helpful, too. - * - * Therefore, we do a quick sanity check. - * - * If compiling Thumb-1 for a target which supports ARM instructions, we will - * emit a warning, as it is not a "sane" platform to compile for. - * - * Usually, if this happens, it is because of an accident and you probably need - * to specify -march, as you likely meant to compile for a newer architecture. - * - * Credit: large sections of the vectorial and asm source code paths - * have been contributed by @easyaspi314 - */ -#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) -# warning "XXH3 is highly inefficient without ARM or Thumb-2." -#endif + #ifndef XXH_NO_XXH3 -/* ========================================== - * Vectorization detection - * ========================================== */ + /* ********************************************************************* + * XXH3 + * New generation hash designed for speed on small keys and vectorization + ************************************************************************ */ + /*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ -#ifdef XXH_DOXYGEN -/*! - * @ingroup tuning - * @brief Overrides the vectorization implementation chosen for XXH3. - * - * Can be defined to 0 to disable SIMD or any of the values mentioned in - * @ref XXH_VECTOR_TYPE. - * - * If this is not defined, it uses predefined macros to determine the best - * implementation. - */ -# define XXH_VECTOR XXH_SCALAR + */ + + /* === Compiler specifics === */ + + #if ((defined(sun) || defined(__sun)) && \ + __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested \ + with GCC 5.5 */ + #define XXH_RESTRICT /* disable */ + #elif defined(__STDC_VERSION__) && \ + __STDC_VERSION__ >= 199901L /* >= C99 */ + #define XXH_RESTRICT restrict + #elif (defined(__GNUC__) && \ + ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) || \ + (defined(__clang__)) || (defined(_MSC_VER) && (_MSC_VER >= 1400)) || \ + (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) + /* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ + #define XXH_RESTRICT __restrict + #else + #define XXH_RESTRICT /* disable */ + #endif + + #if (defined(__GNUC__) && (__GNUC__ >= 3)) || \ + (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || \ + defined(__clang__) + #define XXH_likely(x) __builtin_expect(x, 1) + #define XXH_unlikely(x) __builtin_expect(x, 0) + #else + #define XXH_likely(x) (x) + #define XXH_unlikely(x) (x) + #endif + + #ifndef XXH_HAS_INCLUDE + #ifdef __has_include + /* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ + #define XXH_HAS_INCLUDE __has_include + #else + #define XXH_HAS_INCLUDE(x) 0 + #endif + #endif + + #if defined(__GNUC__) || defined(__clang__) + #if defined(__ARM_FEATURE_SVE) + #include + #endif + #if defined(__ARM_NEON__) || defined(__ARM_NEON) || \ + (defined(_M_ARM) && _M_ARM >= 7) || defined(_M_ARM64) || \ + defined(_M_ARM64EC) || \ + (defined(__wasm_simd128__) && \ + XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ + #define inline __inline__ /* circumvent a clang bug */ + #include + #undef inline + #elif defined(__AVX2__) + #include + #elif defined(__SSE2__) + #include + #endif + #endif + + #if defined(_MSC_VER) + #include + #endif + + /* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on + * 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because + * the shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the + * lower 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and + * therefore, we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be + * efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and + * 64-bit platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is + * one notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic + * need Lo registers, and this shuffling results in thousands more MOVs + * than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 + * registers, do a 32->64 multiply with UMULL, and the flexible operand + * allowing free shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we + * will emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably + * need to specify -march, as you likely meant to compile for a newer + * architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ + #if defined(__thumb__) && !defined(__thumb2__) && \ + defined(__ARM_ARCH_ISA_ARM) + #warning "XXH3 is highly inefficient without ARM or Thumb-2." + #endif + + /* ========================================== + * Vectorization detection + * ========================================== */ + + #ifdef XXH_DOXYGEN + /*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the + * best implementation. + */ + #define XXH_VECTOR XXH_SCALAR /*! * @ingroup tuning * @brief Possible values for @ref XXH_VECTOR. @@ -3807,491 +4232,560 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can * internal macro XXH_X86DISPATCH overrides this. */ enum XXH_VECTOR_TYPE /* fake enum */ { - XXH_SCALAR = 0, /*!< Portable scalar version */ - XXH_SSE2 = 1, /*!< - * SSE2 for Pentium 4, Opteron, all x86_64. - * - * @note SSE2 is also guaranteed on Windows 10, macOS, and - * Android x86. - */ - XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ - XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ - XXH_NEON = 4, /*!< - * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 - * via the SIMDeverywhere polyfill provided with the - * Emscripten SDK. - */ - XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ - XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ -}; -/*! - * @ingroup tuning - * @brief Selects the minimum alignment for XXH3's accumulators. - * - * When using SIMD, this should match the alignment required for said vector - * type, so, for example, 32 for AVX2. - * - * Default: Auto detected. - */ -# define XXH_ACC_ALIGN 8 -#endif - -/* Actual definition */ -#ifndef XXH_DOXYGEN -# define XXH_SCALAR 0 -# define XXH_SSE2 1 -# define XXH_AVX2 2 -# define XXH_AVX512 3 -# define XXH_NEON 4 -# define XXH_VSX 5 -# define XXH_SVE 6 -#endif - -#ifndef XXH_VECTOR /* can be defined on command line */ -# if defined(__ARM_FEATURE_SVE) -# define XXH_VECTOR XXH_SVE -# elif ( \ - defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ - || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ - || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ - ) && ( \ - defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ - || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ - ) -# define XXH_VECTOR XXH_NEON -# elif defined(__AVX512F__) -# define XXH_VECTOR XXH_AVX512 -# elif defined(__AVX2__) -# define XXH_VECTOR XXH_AVX2 -# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) -# define XXH_VECTOR XXH_SSE2 -# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ - || (defined(__s390x__) && defined(__VEC__)) \ - && defined(__GNUC__) /* TODO: IBM XL */ -# define XXH_VECTOR XXH_VSX -# else -# define XXH_VECTOR XXH_SCALAR -# endif -#endif - -/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ -#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) -# ifdef _MSC_VER -# pragma warning(once : 4606) -# else -# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." -# endif -# undef XXH_VECTOR -# define XXH_VECTOR XXH_SCALAR -#endif - -/* - * Controls the alignment of the accumulator, - * for compatibility with aligned vector loads, which are usually faster. - */ -#ifndef XXH_ACC_ALIGN -# if defined(XXH_X86DISPATCH) -# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ -# elif XXH_VECTOR == XXH_SCALAR /* scalar */ -# define XXH_ACC_ALIGN 8 -# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ -# define XXH_ACC_ALIGN 16 -# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ -# define XXH_ACC_ALIGN 32 -# elif XXH_VECTOR == XXH_NEON /* neon */ -# define XXH_ACC_ALIGN 16 -# elif XXH_VECTOR == XXH_VSX /* vsx */ -# define XXH_ACC_ALIGN 16 -# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ -# define XXH_ACC_ALIGN 64 -# elif XXH_VECTOR == XXH_SVE /* sve */ -# define XXH_ACC_ALIGN 64 -# endif -#endif -#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ - || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 -# define XXH_SEC_ALIGN XXH_ACC_ALIGN -#elif XXH_VECTOR == XXH_SVE -# define XXH_SEC_ALIGN XXH_ACC_ALIGN -#else -# define XXH_SEC_ALIGN 8 -#endif - -#if defined(__GNUC__) || defined(__clang__) -# define XXH_ALIASING __attribute__((may_alias)) -#else -# define XXH_ALIASING /* nothing */ -#endif + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ -/* - * UGLY HACK: - * GCC usually generates the best code with -O3 for xxHash. - * - * However, when targeting AVX2, it is overzealous in its unrolling resulting - * in code roughly 3/4 the speed of Clang. - * - * There are other issues, such as GCC splitting _mm256_loadu_si256 into - * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which - * only applies to Sandy and Ivy Bridge... which don't even support AVX2. - * - * That is why when compiling the AVX2 version, it is recommended to use either - * -O2 -mavx2 -march=haswell - * or - * -O2 -mavx2 -mno-avx256-split-unaligned-load - * for decent performance, or to use Clang instead. - * - * Fortunately, we can control the first one with a pragma that forces GCC into - * -O2, but the other one we can't control without "failed to inline always - * inline function due to target mismatch" warnings. - */ -#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ - && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ -# pragma GCC push_options -# pragma GCC optimize("-O2") -#endif +}; -#if XXH_VECTOR == XXH_NEON + /*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said + * vector type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ + #define XXH_ACC_ALIGN 8 + #endif + + /* Actual definition */ + #ifndef XXH_DOXYGEN + #define XXH_SCALAR 0 + #define XXH_SSE2 1 + #define XXH_AVX2 2 + #define XXH_AVX512 3 + #define XXH_NEON 4 + #define XXH_VSX 5 + #define XXH_SVE 6 + #endif + + #ifndef XXH_VECTOR /* can be defined on command line */ + #if defined(__ARM_FEATURE_SVE) + #define XXH_VECTOR XXH_SVE + #elif (defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || \ + defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && \ + XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && \ + (defined(_WIN32) || \ + defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + #define XXH_VECTOR XXH_NEON + #elif defined(__AVX512F__) + #define XXH_VECTOR XXH_AVX512 + #elif defined(__AVX2__) + #define XXH_VECTOR XXH_AVX2 + #elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) + #define XXH_VECTOR XXH_SSE2 + #elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) || \ + (defined(__s390x__) && defined(__VEC__)) && \ + defined(__GNUC__) /* TODO: IBM XL */ + #define XXH_VECTOR XXH_VSX + #else + #define XXH_VECTOR XXH_SCALAR + #endif + #endif + + /* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ + #if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) + #ifdef _MSC_VER + #pragma warning(once : 4606) + #else + #warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." + #endif + #undef XXH_VECTOR + #define XXH_VECTOR XXH_SCALAR + #endif + + /* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ + #ifndef XXH_ACC_ALIGN + #if defined(XXH_X86DISPATCH) + #define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ + #elif XXH_VECTOR == XXH_SCALAR /* scalar */ + #define XXH_ACC_ALIGN 8 + #elif XXH_VECTOR == XXH_SSE2 /* sse2 */ + #define XXH_ACC_ALIGN 16 + #elif XXH_VECTOR == XXH_AVX2 /* avx2 */ + #define XXH_ACC_ALIGN 32 + #elif XXH_VECTOR == XXH_NEON /* neon */ + #define XXH_ACC_ALIGN 16 + #elif XXH_VECTOR == XXH_VSX /* vsx */ + #define XXH_ACC_ALIGN 16 + #elif XXH_VECTOR == XXH_AVX512 /* avx512 */ + #define XXH_ACC_ALIGN 64 + #elif XXH_VECTOR == XXH_SVE /* sve */ + #define XXH_ACC_ALIGN 64 + #endif + #endif + + #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 || \ + XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 + #define XXH_SEC_ALIGN XXH_ACC_ALIGN + #elif XXH_VECTOR == XXH_SVE + #define XXH_SEC_ALIGN XXH_ACC_ALIGN + #else + #define XXH_SEC_ALIGN 8 + #endif + + #if defined(__GNUC__) || defined(__clang__) + #define XXH_ALIASING __attribute__((may_alias)) + #else + #define XXH_ALIASING /* nothing */ + #endif + + /* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling + * resulting in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization + * which only applies to Sandy and Ivy Bridge... which don't even support + * AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use + * either -O2 -mavx2 -march=haswell or -O2 -mavx2 + * -mno-avx256-split-unaligned-load for decent performance, or to use + * Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC + * into -O2, but the other one we can't control without "failed to inline + * always inline function due to target mismatch" warnings. + */ + #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && \ + XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ + #pragma GCC push_options + #pragma GCC optimize("-O2") + #endif + + #if XXH_VECTOR == XXH_NEON /* - * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 - * optimizes out the entire hashLong loop because of the aliasing violation. + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC + * -O3 optimizes out the entire hashLong loop because of the aliasing violation. * * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, * so the only option is to mark it as aliasing. */ typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; -/*! - * @internal - * @brief `vld1q_u64` but faster and alignment-safe. - * - * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only - * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). - * - * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it - * prohibits load-store optimizations. Therefore, a direct dereference is used. - * - * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe - * unaligned load. - */ -#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) -XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ + /*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is + * only *conditionally* safe (`vld1` has an alignment bit like + * `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so + * it prohibits load-store optimizations. Therefore, a direct + * dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a + * safe unaligned load. + */ + #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t +XXH_vld1q_u64(void const *ptr) /* silence -Wcast-align */ { - return *(xxh_aliasing_uint64x2_t const *)ptr; + + return *(xxh_aliasing_uint64x2_t const *)ptr; + } -#else -XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) -{ - return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); + + #else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const *ptr) { + + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const *)ptr)); + } -#endif -/*! - * @internal - * @brief `vmlal_u32` on low and high halves of a vector. - * - * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with - * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` - * with `vmlal_u32`. - */ -#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 -XXH_FORCE_INLINE uint64x2_t -XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) -{ - /* Inline assembly is the only way */ - __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); - return acc; + #endif + + /*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented + * arm_neon.h with inline assembly and were therefore incapable of + * merging the `vget_{low, high}_u32` with `vmlal_u32`. + */ + #if defined(__aarch64__) && defined(__GNUC__) && \ + !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, + uint32x4_t rhs) { + + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w"(acc) : "w"(lhs), "w"(rhs)); + return acc; + } -XXH_FORCE_INLINE uint64x2_t -XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) -{ - /* This intrinsic works as expected */ - return vmlal_high_u32(acc, lhs, rhs); + +XXH_FORCE_INLINE uint64x2_t XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, + uint32x4_t rhs) { + + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); + } -#else + + #else /* Portable intrinsic versions */ -XXH_FORCE_INLINE uint64x2_t -XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) -{ - return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +XXH_FORCE_INLINE uint64x2_t XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, + uint32x4_t rhs) { + + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); + } + /*! @copydoc XXH_vmlal_low_u32 * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ -XXH_FORCE_INLINE uint64x2_t -XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) -{ - return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); -} -#endif +XXH_FORCE_INLINE uint64x2_t XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, + uint32x4_t rhs) { -/*! - * @ingroup tuning - * @brief Controls the NEON to scalar ratio for XXH3 - * - * This can be set to 2, 4, 6, or 8. - * - * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. - * - * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those - * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU - * bandwidth. - * - * This is even more noticeable on the more advanced cores like the Cortex-A76 which - * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. - * - * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes - * and 2 scalar lanes, which is chosen by default. - * - * This does not apply to Apple processors or 32-bit processors, which run better with - * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. - * - * This change benefits CPUs with large micro-op buffers without negatively affecting - * most other CPUs: - * - * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | - * |:----------------------|:--------------------|----------:|-----------:|------:| - * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | - * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | - * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | - * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | - * - * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. - * - * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning - * it effectively becomes worse 4. - * - * @see XXH3_accumulate_512_neon() - */ -# ifndef XXH3_NEON_LANES -# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ - && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 -# define XXH3_NEON_LANES 6 -# else -# define XXH3_NEON_LANES XXH_ACC_NB -# endif -# endif -#endif /* XXH_VECTOR == XXH_NEON */ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); -/* - * VSX and Z Vector helpers. - * - * This is very messy, and any pull requests to clean this up are welcome. - * - * There are a lot of problems with supporting VSX and s390x, due to - * inconsistent intrinsics, spotty coverage, and multiple endiannesses. - */ -#if XXH_VECTOR == XXH_VSX -/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, - * and `pixel`. This is a problem for obvious reasons. - * - * These keywords are unnecessary; the spec literally says they are - * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd - * after including the header. - * - * We use pragma push_macro/pop_macro to keep the namespace clean. */ -# pragma push_macro("bool") -# pragma push_macro("vector") -# pragma push_macro("pixel") -/* silence potential macro redefined warnings */ -# undef bool -# undef vector -# undef pixel +} -# if defined(__s390x__) -# include -# else -# include -# endif + #endif -/* Restore the original macro values, if applicable. */ -# pragma pop_macro("pixel") -# pragma pop_macro("vector") -# pragma pop_macro("bool") + /*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but + * only 2 of those can be NEON. If you are only using NEON instructions, + * you are only using 2/3 of the CPU bandwidth. + * + * This is even more noticeable on the more advanced cores like the + * Cortex-A76 which can dispatch 8 micro-ops per cycle, but still only 2 + * NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to + * run 6 NEON lanes and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which + * run better with full NEON. These will default to 8. Additionally, + * size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without + * negatively affecting most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 + * hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 + * GB/s | ~16% | | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 + * GB/s | 5.3 GB/s | ~5% | | Marvell PXA1928 (A53) | In-order + * dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | | Apple M1 | 4 NEON/8 + * micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as + * fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of + * the lanes meaning it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ + #ifndef XXH3_NEON_LANES + #if (defined(__aarch64__) || defined(__arm64__) || \ + defined(_M_ARM64) || defined(_M_ARM64EC)) && \ + !defined(__APPLE__) && XXH_SIZE_OPT <= 0 + #define XXH3_NEON_LANES 6 + #else + #define XXH3_NEON_LANES XXH_ACC_NB + #endif + #endif + #endif /* XXH_VECTOR == XXH_NEON */ + + /* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ + #if XXH_VECTOR == XXH_VSX + /* Annoyingly, these headers _may_ define three macros: `bool`, + * `vector`, and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ + #pragma push_macro("bool") + #pragma push_macro("vector") + #pragma push_macro("pixel") + /* silence potential macro redefined warnings */ + #undef bool + #undef vector + #undef pixel + + #if defined(__s390x__) + #include + #else + #include + #endif + + /* Restore the original macro values, if applicable. */ + #pragma pop_macro("pixel") + #pragma pop_macro("vector") + #pragma pop_macro("bool") typedef __vector unsigned long long xxh_u64x2; -typedef __vector unsigned char xxh_u8x16; -typedef __vector unsigned xxh_u32x4; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; /* - * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing + * issue. */ typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; -# ifndef XXH_VSX_BE -# if defined(__BIG_ENDIAN__) \ - || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -# define XXH_VSX_BE 1 -# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ -# warning "-maltivec=be is not recommended. Please use native endianness." -# define XXH_VSX_BE 1 -# else -# define XXH_VSX_BE 0 -# endif -# endif /* !defined(XXH_VSX_BE) */ - -# if XXH_VSX_BE -# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) -# define XXH_vec_revb vec_revb -# else + #ifndef XXH_VSX_BE + #if defined(__BIG_ENDIAN__) || \ + (defined(__BYTE_ORDER__) && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define XXH_VSX_BE 1 + #elif defined(__VEC_ELEMENT_REG_ORDER__) && \ + __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ + #warning \ + "-maltivec=be is not recommended. Please use native endianness." + #define XXH_VSX_BE 1 + #else + #define XXH_VSX_BE 0 + #endif + #endif /* !defined(XXH_VSX_BE) */ + + #if XXH_VSX_BE + #if defined(__POWER9_VECTOR__) || \ + (defined(__clang__) && defined(__s390x__)) + #define XXH_vec_revb vec_revb + #else /*! * A polyfill for POWER9's vec_revb(). */ -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) -{ - xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, - 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; - return vec_perm(val, val, vByteSwap); +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) { + + xxh_u8x16 const vByteSwap = {0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08}; + return vec_perm(val, val, vByteSwap); + } -# endif -# endif /* XXH_VSX_BE */ + + #endif + #endif /* XXH_VSX_BE */ /*! * Performs an unaligned vector load and byte swaps it on big endian. */ -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) -{ - xxh_u64x2 ret; - XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); -# if XXH_VSX_BE - ret = XXH_vec_revb(ret); -# endif - return ret; -} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) { -/* - * vec_mulo and vec_mule are very problematic intrinsics on PowerPC - * - * These intrinsics weren't added until GCC 8, despite existing for a while, - * and they are endian dependent. Also, their meaning swap depending on version. - * */ -# if defined(__s390x__) - /* s390x is always big endian, no issue on this platform */ -# define XXH_vec_mulo vec_mulo -# define XXH_vec_mule vec_mule -# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) -/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ - /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ -# define XXH_vec_mulo __builtin_altivec_vmulouw -# define XXH_vec_mule __builtin_altivec_vmuleuw -# else -/* gcc needs inline assembly */ -/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) -{ - xxh_u64x2 result; - __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); - return result; -} -XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) -{ - xxh_u64x2 result; - __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); - return result; -} -# endif /* XXH_vec_mulo, XXH_vec_mule */ -#endif /* XXH_VECTOR == XXH_VSX */ - -#if XXH_VECTOR == XXH_SVE -#define ACCRND(acc, offset) \ -do { \ - svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ - svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ - svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ - svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ - svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ - svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ - svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ - acc = svadd_u64_x(mask, acc, mul); \ -} while (0) -#endif /* XXH_VECTOR == XXH_SVE */ - -/* prefetch - * can be disabled, by declaring XXH_NO_PREFETCH build macro */ -#if defined(XXH_NO_PREFETCH) -# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ -#else -# if XXH_SIZE_OPT >= 1 -# define XXH_PREFETCH(ptr) (void)(ptr) -# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ -# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ -# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) -# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) -# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) -# else -# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ -# endif -#endif /* XXH_NO_PREFETCH */ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); + #if XXH_VSX_BE + ret = XXH_vec_revb(ret); + #endif + return ret; +} -/* ========================================== - * XXH3 default settings - * ========================================== */ - -#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ - -#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) -# error "default keyset is not large enough" -#endif + /* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a + * while, and they are endian dependent. Also, their meaning swap + * depending on version. + * */ + #if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ + #define XXH_vec_mulo vec_mulo + #define XXH_vec_mule vec_mule + #elif defined(__clang__) && \ + XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) + /* Clang has a better way to control this, we can just use the builtin + * which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the + * vec_* operations */ + #define XXH_vec_mulo __builtin_altivec_vmulouw + #define XXH_vec_mule __builtin_altivec_vmuleuw + #else +/* gcc needs inline assembly */ +/* Adapted from + * https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) { + + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; + +} + +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) { + + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; + +} + + #endif /* XXH_vec_mulo, XXH_vec_mule */ + #endif /* XXH_VECTOR == XXH_VSX */ + + #if XXH_VECTOR == XXH_SVE + #define ACCRND(acc, offset) \ + do { \ + \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ + \ + } while (0) + #endif /* XXH_VECTOR == XXH_SVE */ + + /* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ + #if defined(XXH_NO_PREFETCH) + #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ + #else + #if XXH_SIZE_OPT >= 1 + #define XXH_PREFETCH(ptr) (void)(ptr) + #elif defined(_MSC_VER) && \ + (defined(_M_X64) || \ + defined( \ + _M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ + #include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ + #define XXH_PREFETCH(ptr) \ + _mm_prefetch((const char *)(ptr), _MM_HINT_T0) + #elif defined(__GNUC__) && \ + ((__GNUC__ >= 4) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))) + #define XXH_PREFETCH(ptr) \ + __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) + #else + #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ + #endif + #endif /* XXH_NO_PREFETCH */ + + /* ========================================== + * XXH3 default settings + * ========================================== */ + + #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + + #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) + #error "default keyset is not large enough" + #endif /*! Pseudorandom secret taken directly from FARSH. */ -XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { - 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, - 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, - 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, - 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, - 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, - 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, - 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, - 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, - 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, - 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, - 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, - 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +XXH_ALIGN(64) +static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, + 0xf7, 0x21, 0xad, 0x1c, 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, + 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, 0xcb, 0x79, 0xe6, 0x4e, + 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, + 0x81, 0x3a, 0x26, 0x4c, 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, + 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, 0x71, 0x64, 0x48, 0x97, + 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, + 0xc7, 0x0b, 0x4f, 0x1d, 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, + 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, 0xea, 0xc5, 0xac, 0x83, + 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, + 0x29, 0xd4, 0x68, 0x9e, 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, + 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, 0x45, 0xcb, 0x3a, 0x8f, + 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, + }; -static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ -static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< + 0b0001011001010110011001111001000110011110001101110111100111111001 + */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< + 0b1001111110110010000111000110010100011110100110001101111100100101 + */ -#ifdef XXH_OLD_NAMES -# define kSecret XXH3_kSecret -#endif + #ifdef XXH_OLD_NAMES + #define kSecret XXH3_kSecret + #endif -#ifdef XXH_DOXYGEN + #ifdef XXH_DOXYGEN /*! * @brief Calculates a 32-bit to 64-bit long multiply. * * Implemented as a macro. * - * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't - * need to (but it shouldn't need to anyways, it is about 7 instructions to do - * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we - * use that instead of the normal method. + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it + * doesn't need to (but it shouldn't need to anyways, it is about 7 instructions + * to do a 64x64 multiply...). Since we know that this will _always_ emit + * `MULL`, we use that instead of the normal method. * - * If you are compiling for platforms like Thumb-1 and don't have a better option, - * you may also want to write your own long multiply routine here. + * If you are compiling for platforms like Thumb-1 and don't have a better + * option, you may also want to write your own long multiply routine here. * * @param x, y Numbers to be multiplied * @return 64-bit product of the low 32 bits of @p x and @p y. */ -XXH_FORCE_INLINE xxh_u64 -XXH_mult32to64(xxh_u64 x, xxh_u64 y) -{ - return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { + + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); + } -#elif defined(_MSC_VER) && defined(_M_IX86) -# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) -#else -/* - * Downcast + upcast is usually better than masking on older compilers like - * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. - * - * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands - * and perform a full 64x64 multiply -- entirely redundant on 32-bit. - */ -# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) -#endif + + #elif defined(_MSC_VER) && defined(_M_IX86) + #define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) + #else + /* + * Downcast + upcast is usually better than masking on older compilers + * like GCC 4.2 (especially 32-bit ones), all without affecting newer + * compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both + * operands and perform a full 64x64 multiply -- entirely redundant on + * 32-bit. + */ + #define XXH_mult32to64(x, y) \ + ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) + #endif /*! * @brief Calculates a 64->128-bit long multiply. @@ -4302,164 +4796,170 @@ XXH_mult32to64(xxh_u64 x, xxh_u64 y) * @param lhs , rhs The 64-bit integers to be multiplied * @return The 128-bit result represented in an @ref XXH128_hash_t. */ -static XXH128_hash_t -XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) -{ - /* - * GCC/Clang __uint128_t method. - * - * On most 64-bit targets, GCC and Clang define a __uint128_t type. - * This is usually the best way as it usually uses a native long 64-bit - * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. - * - * Usually. - * - * Despite being a 32-bit platform, Clang (and emscripten) define this type - * despite not having the arithmetic for it. This results in a laggy - * compiler builtin call which calculates a full 128-bit multiply. - * In that case it is best to use the portable one. - * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 - */ -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ - && defined(__SIZEOF_INT128__) \ - || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) +static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { + + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this + * type despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ + #if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) && \ + defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; - __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; - XXH128_hash_t r128; - r128.low64 = (xxh_u64)(product); - r128.high64 = (xxh_u64)(product >> 64); - return r128; - - /* - * MSVC for x64's _umul128 method. - * - * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); - * - * This compiles to single operand MUL on x64. - */ -#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) - -#ifndef _MSC_VER -# pragma intrinsic(_umul128) -#endif - xxh_u64 product_high; - xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); - XXH128_hash_t r128; - r128.low64 = product_low; - r128.high64 = product_high; - return r128; - - /* - * MSVC for ARM64's __umulh method. - * - * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. - */ -#elif defined(_M_ARM64) || defined(_M_ARM64EC) - -#ifndef _MSC_VER -# pragma intrinsic(__umulh) -#endif - XXH128_hash_t r128; - r128.low64 = lhs * rhs; - r128.high64 = __umulh(lhs, rhs); - return r128; + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 + * *HighProduct); + * + * This compiles to single operand MUL on x64. + */ + #elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + + #ifndef _MSC_VER + #pragma intrinsic(_umul128) + #endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; -#else - /* - * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. - * - * This is a fast and simple grade school multiply, which is shown below - * with base 10 arithmetic instead of base 0x100000000. - * - * 9 3 // D2 lhs = 93 - * x 7 5 // D2 rhs = 75 - * ---------- - * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 - * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 - * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 - * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 - * --------- - * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 - * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 - * --------- - * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 - * - * The reasons for adding the products like this are: - * 1. It avoids manual carry tracking. Just like how - * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. - * This avoids a lot of complexity. - * - * 2. It hints for, and on Clang, compiles to, the powerful UMAAL - * instruction available in ARM's Digital Signal Processing extension - * in 32-bit ARMv6 and later, which is shown below: - * - * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) - * { - * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; - * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); - * *RdHi = (xxh_u32)(product >> 32); - * } - * - * This instruction was designed for efficient long multiplication, and - * allows this to be calculated in only 4 instructions at speeds - * comparable to some 64-bit ALUs. - * - * 3. It isn't terrible on other platforms. Usually this will be a couple - * of 32-bit ADD/ADCs. - */ + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t + * method. + */ + #elif defined(_M_ARM64) || defined(_M_ARM64EC) + + #ifndef _MSC_VER + #pragma intrinsic(__umulh) + #endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + + #else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; + #endif - /* First calculate all of the cross products. */ - xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); - xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); - xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); - xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); - - /* Now add the products together. These will never overflow. */ - xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; - xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; - xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); - - XXH128_hash_t r128; - r128.low64 = lower; - r128.high64 = upper; - return r128; -#endif } /*! * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. * * The reason for the separate function is to prevent passing too many structs - * around by value. This will hopefully inline the multiply, but we don't force it. + * around by value. This will hopefully inline the multiply, but we don't force + * it. * * @param lhs , rhs The 64-bit integers to multiply * @return The low 64 bits of the product XOR'd by the high 64 bits. * @see XXH_mult64to128() */ -static xxh_u64 -XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) -{ - XXH128_hash_t product = XXH_mult64to128(lhs, rhs); - return product.low64 ^ product.high64; +static xxh_u64 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { + + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; + } /*! Seems to produce slightly better code on GCC for some reason. */ -XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) -{ - XXH_ASSERT(0 <= shift && shift < 64); - return v64 ^ (v64 >> shift); +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) { + + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); + } /* * This is a fast avalanche stage, * suitable when input bits are already partially mixed */ -static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) -{ - h64 = XXH_xorshift64(h64, 37); - h64 *= PRIME_MX1; - h64 = XXH_xorshift64(h64, 32); - return h64; +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) { + + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; + } /* @@ -4467,16 +4967,16 @@ static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) * inspired by Pelle Evensen's rrmxmx * preferable when input has not been previously mixed */ -static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) -{ - /* this mix is inspired by Pelle Evensen's rrmxmx */ - h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); - h64 *= PRIME_MX2; - h64 ^= (h64 >> 35) + len ; - h64 *= PRIME_MX2; - return XXH_xorshift64(h64, 28); -} +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) { + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); + +} /* ========================================== * Short keys @@ -4486,7 +4986,8 @@ static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) * favored lengths that were a multiple of 4 or 8. * * Instead of iterating over individual inputs, we use a set of single shot - * functions which piece together a range of lengths and operate in constant time. + * functions which piece together a range of lengths and operate in constant + * time. * * Additionally, the number of multiplies has been significantly reduced. This * reduces latency, especially when emulating 64-bit multiplies on 32-bit. @@ -4511,70 +5012,100 @@ static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) * * This adds an extra layer of strength for custom secrets. */ -XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t -XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(1 <= len && len <= 3); - XXH_ASSERT(secret != NULL); - /* - * len = 1: combined = { input[0], 0x01, input[0], input[0] } - * len = 2: combined = { input[1], 0x02, input[0], input[1] } - * len = 3: combined = { input[2], 0x03, input[0], input[1] } - */ - { xxh_u8 const c1 = input[0]; - xxh_u8 const c2 = input[len >> 1]; - xxh_u8 const c3 = input[len - 1]; - xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) - | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); - xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; - xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; - return XXH64_avalanche(keyed); - } +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_1to3_64b(const xxh_u8 *input, + size_t len, + const xxh_u8 *secret, + XXH64_hash_t seed) { + + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { + + xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) | + ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = + (XXH_readLE32(secret) ^ XXH_readLE32(secret + 4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + + } + } -XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t -XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(4 <= len && len <= 8); - seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; - { xxh_u32 const input1 = XXH_readLE32(input); - xxh_u32 const input2 = XXH_readLE32(input + len - 4); - xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; - xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); - xxh_u64 const keyed = input64 ^ bitflip; - return XXH3_rrmxmx(keyed, len); - } +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_4to8_64b(const xxh_u8 *input, + size_t len, + const xxh_u8 *secret, + XXH64_hash_t seed) { + + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { + + xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = + (XXH_readLE64(secret + 8) ^ XXH_readLE64(secret + 16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + + } + } -XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t -XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(9 <= len && len <= 16); - { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; - xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; - xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; - xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; - xxh_u64 const acc = len - + XXH_swap64(input_lo) + input_hi - + XXH3_mul128_fold64(input_lo, input_hi); - return XXH3_avalanche(acc); - } +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_9to16_64b(const xxh_u8 *input, + size_t len, + const xxh_u8 *secret, + XXH64_hash_t seed) { + + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { + + xxh_u64 const bitflip1 = + (XXH_readLE64(secret + 24) ^ XXH_readLE64(secret + 32)) + seed; + xxh_u64 const bitflip2 = + (XXH_readLE64(secret + 40) ^ XXH_readLE64(secret + 48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + + } + } -XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t -XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(len <= 16); - { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); - if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); - if (len) return XXH3_len_1to3_64b(input, len, secret, seed); - return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); - } +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_0to16_64b(const xxh_u8 *input, + size_t len, + const xxh_u8 *secret, + XXH64_hash_t seed) { + + XXH_ASSERT(len <= 16); + { + + if (XXH_likely(len > 8)) + return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) + return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche( + seed ^ (XXH_readLE64(secret + 56) ^ XXH_readLE64(secret + 64))); + + } + } /* @@ -4603,106 +5134,134 @@ XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_ * by this, although it is always a good idea to use a proper seed if you care * about strength. */ -XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, - const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) -{ -#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ - /* - * UGLY HACK: - * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in - * slower code. - * - * By forcing seed64 into a register, we disrupt the cost model and - * cause it to scalarize. See `XXH32_round()` - * - * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, - * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on - * GCC 9.2, despite both emitting scalar code. - * - * GCC generates much better scalar code than Clang for the rest of XXH3, - * which is why finding a more optimal codepath is an interest. - */ - XXH_COMPILER_GUARD(seed64); -#endif - { xxh_u64 const input_lo = XXH_readLE64(input); - xxh_u64 const input_hi = XXH_readLE64(input+8); - return XXH3_mul128_fold64( - input_lo ^ (XXH_readLE64(secret) + seed64), - input_hi ^ (XXH_readLE64(secret+8) - seed64) - ); - } +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8 *XXH_RESTRICT input, + const xxh_u8 *XXH_RESTRICT secret, + xxh_u64 seed64) { + + #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like \ + XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); + #endif + { + + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input + 8); + return XXH3_mul128_fold64(input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret + 8) - seed64)); + + } + } /* For mid range keys, XXH3 uses a Mum-hash variant. */ -XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t -XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH64_hash_t seed) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; - XXH_ASSERT(16 < len && len <= 128); - - { xxh_u64 acc = len * XXH_PRIME64_1; -#if XXH_SIZE_OPT >= 1 - /* Smaller and cleaner, but slightly slower. */ - unsigned int i = (unsigned int)(len - 1) / 32; - do { - acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); - acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); - } while (i-- != 0); -#else - if (len > 32) { - if (len > 64) { - if (len > 96) { - acc += XXH3_mix16B(input+48, secret+96, seed); - acc += XXH3_mix16B(input+len-64, secret+112, seed); - } - acc += XXH3_mix16B(input+32, secret+64, seed); - acc += XXH3_mix16B(input+len-48, secret+80, seed); - } - acc += XXH3_mix16B(input+16, secret+32, seed); - acc += XXH3_mix16B(input+len-32, secret+48, seed); +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_17to128_64b( + const xxh_u8 *XXH_RESTRICT input, size_t len, + const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { + + xxh_u64 acc = len * XXH_PRIME64_1; + #if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + + acc += XXH3_mix16B(input + 16 * i, secret + 32 * i, seed); + acc += + XXH3_mix16B(input + len - 16 * (i + 1), secret + 32 * i + 16, seed); + + } while (i-- != 0); + + #else + if (len > 32) { + + if (len > 64) { + + if (len > 96) { + + acc += XXH3_mix16B(input + 48, secret + 96, seed); + acc += XXH3_mix16B(input + len - 64, secret + 112, seed); + } - acc += XXH3_mix16B(input+0, secret+0, seed); - acc += XXH3_mix16B(input+len-16, secret+16, seed); -#endif - return XXH3_avalanche(acc); + + acc += XXH3_mix16B(input + 32, secret + 64, seed); + acc += XXH3_mix16B(input + len - 48, secret + 80, seed); + + } + + acc += XXH3_mix16B(input + 16, secret + 32, seed); + acc += XXH3_mix16B(input + len - 32, secret + 48, seed); + } + + acc += XXH3_mix16B(input + 0, secret + 0, seed); + acc += XXH3_mix16B(input + len - 16, secret + 16, seed); + #endif + return XXH3_avalanche(acc); + + } + } -/*! - * @brief Maximum size of "short" key in bytes. - */ -#define XXH3_MIDSIZE_MAX 240 + /*! + * @brief Maximum size of "short" key in bytes. + */ + #define XXH3_MIDSIZE_MAX 240 -XXH_NO_INLINE XXH_PUREF XXH64_hash_t -XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH64_hash_t seed) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; +XXH_NO_INLINE XXH_PUREF XXH64_hash_t XXH3_len_129to240_64b( + const xxh_u8 *XXH_RESTRICT input, size_t len, + const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { + + xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i = 0; i < 8; i++) { - #define XXH3_MIDSIZE_STARTOFFSET 3 - #define XXH3_MIDSIZE_LASTOFFSET 17 + acc += XXH3_mix16B(input + (16 * i), secret + (16 * i), seed); - { xxh_u64 acc = len * XXH_PRIME64_1; - xxh_u64 acc_end; - unsigned int const nbRounds = (unsigned int)len / 16; - unsigned int i; - XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); - for (i=0; i<8; i++) { - acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); - } - /* last bytes */ - acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); - XXH_ASSERT(nbRounds >= 8); - acc = XXH3_avalanche(acc); -#if defined(__clang__) /* Clang */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + } + + /* last bytes */ + acc_end = XXH3_mix16B( + input + len - 16, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); + #if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ /* * UGLY HACK: * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. @@ -4724,441 +5283,522 @@ XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, * SLP vectorization. */ #pragma clang loop vectorize(disable) -#endif - for (i=8 ; i < nbRounds; i++) { - /* - * Prevents clang for unrolling the acc loop and interleaving with this one. - */ - XXH_COMPILER_GUARD(acc); - acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); - } - return XXH3_avalanche(acc + acc_end); + #endif + for (i = 8; i < nbRounds; i++) { + + /* + * Prevents clang for unrolling the acc loop and interleaving with this + * one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += + XXH3_mix16B(input + (16 * i), + secret + (16 * (i - 8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } -} + return XXH3_avalanche(acc + acc_end); + + } + +} + + /* ======= Long Keys ======= */ + + #define XXH_STRIPE_LEN 64 + #define XXH_SECRET_CONSUME_RATE \ + 8 /* nb of secret bytes consumed at each accumulation */ + #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + + #ifdef XXH_OLD_NAMES + #define STRIPE_LEN XXH_STRIPE_LEN + #define ACC_NB XXH_ACC_NB + #endif + + #ifndef XXH_PREFETCH_DIST + #ifdef __clang__ + #define XXH_PREFETCH_DIST 320 + #else + #if (XXH_VECTOR == XXH_AVX512) + #define XXH_PREFETCH_DIST 512 + #else + #define XXH_PREFETCH_DIST 384 + #endif + #endif /* __clang__ */ + #endif /* XXH_PREFETCH_DIST */ + + /* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails + * to optimize the inline function. + */ + #define XXH3_ACCUMULATE_TEMPLATE(name) \ + void XXH3_accumulate_##name( \ + xxh_u64 *XXH_RESTRICT acc, const xxh_u8 *XXH_RESTRICT input, \ + const xxh_u8 *XXH_RESTRICT secret, size_t nbStripes) { \ + \ + size_t n; \ + for (n = 0; n < nbStripes; n++) { \ + \ + const xxh_u8 *const in = input + n * XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name(acc, in, \ + secret + n * XXH_SECRET_CONSUME_RATE); \ + \ + } \ + \ + } -/* ======= Long Keys ======= */ +XXH_FORCE_INLINE void XXH_writeLE64(void *dst, xxh_u64 v64) { -#define XXH_STRIPE_LEN 64 -#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ -#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); -#ifdef XXH_OLD_NAMES -# define STRIPE_LEN XXH_STRIPE_LEN -# define ACC_NB XXH_ACC_NB -#endif +} -#ifndef XXH_PREFETCH_DIST -# ifdef __clang__ -# define XXH_PREFETCH_DIST 320 -# else -# if (XXH_VECTOR == XXH_AVX512) -# define XXH_PREFETCH_DIST 512 -# else -# define XXH_PREFETCH_DIST 384 -# endif -# endif /* __clang__ */ -#endif /* XXH_PREFETCH_DIST */ + /* Several intrinsic functions below are supposed to accept __int64 as + * argument, as documented in + * https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ + #if !defined(__VMS) && \ + (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 199901L) /* C99 */)) +typedef int64_t xxh_i64; + #else +/* the following type must have a width of 64-bit */ +typedef long long xxh_i64; + #endif -/* - * These macros are to generate an XXH3_accumulate() function. - * The two arguments select the name suffix and target attribute. - * - * The name of this symbol is XXH3_accumulate_() and it calls - * XXH3_accumulate_512_(). - * - * It may be useful to hand implement this function if the compiler fails to - * optimize the inline function. - */ -#define XXH3_ACCUMULATE_TEMPLATE(name) \ -void \ -XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ - const xxh_u8* XXH_RESTRICT input, \ - const xxh_u8* XXH_RESTRICT secret, \ - size_t nbStripes) \ -{ \ - size_t n; \ - for (n = 0; n < nbStripes; n++ ) { \ - const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ - XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ - XXH3_accumulate_512_##name( \ - acc, \ - in, \ - secret + n*XXH_SECRET_CONSUME_RATE); \ - } \ -} - - -XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) -{ - if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); - XXH_memcpy(dst, &v64, sizeof(v64)); -} - -/* Several intrinsic functions below are supposed to accept __int64 as argument, - * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . - * However, several environments do not define __int64 type, - * requiring a workaround. - */ -#if !defined (__VMS) \ - && (defined (__cplusplus) \ - || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) - typedef int64_t xxh_i64; -#else - /* the following type must have a width of 64-bit */ - typedef long long xxh_i64; -#endif + /* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the + * most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as + * the product. + * + * This means that in the (relatively likely) case of a multiply by zero, + * the original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together + * in the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + #if (XXH_VECTOR == XXH_AVX512) || \ + (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) -/* - * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. - * - * It is a hardened version of UMAC, based off of FARSH's implementation. - * - * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD - * implementations, and it is ridiculously fast. - * - * We harden it by mixing the original input to the accumulators as well as the product. - * - * This means that in the (relatively likely) case of a multiply by zero, the - * original input is preserved. - * - * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve - * cross-pollination, as otherwise the upper and lower halves would be - * essentially independent. - * - * This doesn't matter on 64-bit hashes since they all get merged together in - * the end, so we skip the extra step. - * - * Both XXH3_64bits and XXH3_128bits use this subroutine. - */ + #ifndef XXH_TARGET_AVX512 + #define XXH_TARGET_AVX512 /* disable attribute target */ + #endif -#if (XXH_VECTOR == XXH_AVX512) \ - || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) +XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_accumulate_512_avx512( + void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, + const void *XXH_RESTRICT secret) { -#ifndef XXH_TARGET_AVX512 -# define XXH_TARGET_AVX512 /* disable attribute target */ -#endif + __m512i *const xacc = (__m512i *)acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); -XXH_FORCE_INLINE XXH_TARGET_AVX512 void -XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, - const void* XXH_RESTRICT input, - const void* XXH_RESTRICT secret) -{ - __m512i* const xacc = (__m512i *) acc; - XXH_ASSERT((((size_t)acc) & 63) == 0); - XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { + + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512(input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512(secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512(data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64(data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32(data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = + _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + + } - { - /* data_vec = input[0]; */ - __m512i const data_vec = _mm512_loadu_si512 (input); - /* key_vec = secret[0]; */ - __m512i const key_vec = _mm512_loadu_si512 (secret); - /* data_key = data_vec ^ key_vec; */ - __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); - /* data_key_lo = data_key >> 32; */ - __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); - /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ - __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); - /* xacc[0] += swap(data_vec); */ - __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); - __m512i const sum = _mm512_add_epi64(*xacc, data_swap); - /* xacc[0] += product; */ - *xacc = _mm512_add_epi64(product, sum); - } } + XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) -/* - * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. - * - * Multiplication isn't perfect, as explained by Google in HighwayHash: - * - * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to - * // varying degrees. In descending order of goodness, bytes - * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. - * // As expected, the upper and lower bytes are much worse. - * - * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 - * - * Since our algorithm uses a pseudorandom secret to add some variance into the - * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. - * - * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid - * extraction. - * - * Both XXH3_64bits and XXH3_128bits use this subroutine. - */ + /* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: + * https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into + * the mix, we don't need to (or want to) mix as often or as much as + * HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to + * avoid extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + + XXH_FORCE_INLINE XXH_TARGET_AVX512 + void XXH3_scrambleAcc_avx512(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { + + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { + + __m512i *const xacc = (__m512i *)acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64(acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512(secret); + __m512i const data_key = _mm512_ternarylogic_epi32( + key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64(data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32(data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32(data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + + } -XXH_FORCE_INLINE XXH_TARGET_AVX512 void -XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) -{ - XXH_ASSERT((((size_t)acc) & 63) == 0); - XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); - { __m512i* const xacc = (__m512i*) acc; - const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); - - /* xacc[0] ^= (xacc[0] >> 47) */ - __m512i const acc_vec = *xacc; - __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); - /* xacc[0] ^= secret; */ - __m512i const key_vec = _mm512_loadu_si512 (secret); - __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); - - /* xacc[0] *= XXH_PRIME32_1; */ - __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); - __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); - __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); - *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); - } } -XXH_FORCE_INLINE XXH_TARGET_AVX512 void -XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) -{ - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); - XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); - XXH_ASSERT(((size_t)customSecret & 63) == 0); - (void)(&XXH_writeLE64); - { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); - __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); - __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); - - const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); - __m512i* const dest = ( __m512i*) customSecret; - int i; - XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ - XXH_ASSERT(((size_t)dest & 63) == 0); - for (i=0; i < nbRounds; ++i) { - dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); - } } +XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512( + void *XXH_RESTRICT customSecret, xxh_u64 seed64) { + + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { + + int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = + _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i *const src = (const __m512i *)((const void *)XXH3_kSecret); + __m512i *const dest = (__m512i *)customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i = 0; i < nbRounds; ++i) { + + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + + } + + } + } -#endif + #endif -#if (XXH_VECTOR == XXH_AVX2) \ - || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + #if (XXH_VECTOR == XXH_AVX2) || \ + (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) -#ifndef XXH_TARGET_AVX2 -# define XXH_TARGET_AVX2 /* disable attribute target */ -#endif + #ifndef XXH_TARGET_AVX2 + #define XXH_TARGET_AVX2 /* disable attribute target */ + #endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2( + void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, + const void *XXH_RESTRICT secret) { + + XXH_ASSERT((((size_t)acc) & 31) == 0); + { + + __m256i *const xacc = (__m256i *)acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. + */ + const __m256i *const xinput = (const __m256i *)input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i *const xsecret = (const __m256i *)secret; + + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { + + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256(xinput + i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256(xsecret + i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256(data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64(data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32(data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = + _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + + } + + } -XXH_FORCE_INLINE XXH_TARGET_AVX2 void -XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, - const void* XXH_RESTRICT input, - const void* XXH_RESTRICT secret) -{ - XXH_ASSERT((((size_t)acc) & 31) == 0); - { __m256i* const xacc = (__m256i *) acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ - const __m256i* const xinput = (const __m256i *) input; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ - const __m256i* const xsecret = (const __m256i *) secret; - - size_t i; - for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { - /* data_vec = xinput[i]; */ - __m256i const data_vec = _mm256_loadu_si256 (xinput+i); - /* key_vec = xsecret[i]; */ - __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); - /* data_key = data_vec ^ key_vec; */ - __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); - /* data_key_lo = data_key >> 32; */ - __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); - /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ - __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); - /* xacc[i] += swap(data_vec); */ - __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); - __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); - /* xacc[i] += product; */ - xacc[i] = _mm256_add_epi64(product, sum); - } } } + XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) -XXH_FORCE_INLINE XXH_TARGET_AVX2 void -XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) -{ - XXH_ASSERT((((size_t)acc) & 31) == 0); - { __m256i* const xacc = (__m256i*) acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ - const __m256i* const xsecret = (const __m256i *) secret; - const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); - - size_t i; - for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { - /* xacc[i] ^= (xacc[i] >> 47) */ - __m256i const acc_vec = xacc[i]; - __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); - __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); - /* xacc[i] ^= xsecret; */ - __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); - __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); - - /* xacc[i] *= XXH_PRIME32_1; */ - __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); - __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); - __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); - xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); - } + XXH_FORCE_INLINE XXH_TARGET_AVX2 + void XXH3_scrambleAcc_avx2(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { + + XXH_ASSERT((((size_t)acc) & 31) == 0); + { + + __m256i *const xacc = (__m256i *)acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i *const xsecret = (const __m256i *)secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { + + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64(acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256(acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256(xsecret + i); + __m256i const data_key = _mm256_xor_si256(data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64(data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32(data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32(data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + + } + } -XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) -{ - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); - XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); - (void)(&XXH_writeLE64); - XXH_PREFETCH(customSecret); - { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2( + void *XXH_RESTRICT customSecret, xxh_u64 seed64) { - const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); - __m256i* dest = ( __m256i*) customSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { + + __m256i const seed = + _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, + (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i *const src = (const __m256i *)((const void *)XXH3_kSecret); + __m256i *dest = (__m256i *)customSecret; + + #if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); + #endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src + 0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src + 1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src + 2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src + 3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src + 4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src + 5), seed); + + } -# if defined(__GNUC__) || defined(__clang__) - /* - * On GCC & Clang, marking 'dest' as modified will cause the compiler: - * - do not extract the secret from sse registers in the internal loop - * - use less common registers, and avoid pushing these reg into stack - */ - XXH_COMPILER_GUARD(dest); -# endif - XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ - XXH_ASSERT(((size_t)dest & 31) == 0); - - /* GCC -O2 need unroll loop manually */ - dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); - dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); - dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); - dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); - dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); - dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); - } } -#endif + #endif -/* x86dispatch always generates SSE2 */ -#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + /* x86dispatch always generates SSE2 */ + #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) -#ifndef XXH_TARGET_SSE2 -# define XXH_TARGET_SSE2 /* disable attribute target */ -#endif + #ifndef XXH_TARGET_SSE2 + #define XXH_TARGET_SSE2 /* disable attribute target */ + #endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_accumulate_512_sse2( + void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, + const void *XXH_RESTRICT secret) { + + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + + __m128i *const xacc = (__m128i *)acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i *const xinput = (const __m128i *)input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i *const xsecret = (const __m128i *)secret; + + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { + + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128(xinput + i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128(xsecret + i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128(data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = + _mm_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32(data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = + _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + + } + + } -XXH_FORCE_INLINE XXH_TARGET_SSE2 void -XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, - const void* XXH_RESTRICT input, - const void* XXH_RESTRICT secret) -{ - /* SSE2 is just a half-scale version of the AVX2 version. */ - XXH_ASSERT((((size_t)acc) & 15) == 0); - { __m128i* const xacc = (__m128i *) acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ - const __m128i* const xinput = (const __m128i *) input; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ - const __m128i* const xsecret = (const __m128i *) secret; - - size_t i; - for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { - /* data_vec = xinput[i]; */ - __m128i const data_vec = _mm_loadu_si128 (xinput+i); - /* key_vec = xsecret[i]; */ - __m128i const key_vec = _mm_loadu_si128 (xsecret+i); - /* data_key = data_vec ^ key_vec; */ - __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); - /* data_key_lo = data_key >> 32; */ - __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); - /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ - __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); - /* xacc[i] += swap(data_vec); */ - __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); - __m128i const sum = _mm_add_epi64(xacc[i], data_swap); - /* xacc[i] += product; */ - xacc[i] = _mm_add_epi64(product, sum); - } } } + XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) -XXH_FORCE_INLINE XXH_TARGET_SSE2 void -XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) -{ - XXH_ASSERT((((size_t)acc) & 15) == 0); - { __m128i* const xacc = (__m128i*) acc; - /* Unaligned. This is mainly for pointer arithmetic, and because - * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ - const __m128i* const xsecret = (const __m128i *) secret; - const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); - - size_t i; - for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { - /* xacc[i] ^= (xacc[i] >> 47) */ - __m128i const acc_vec = xacc[i]; - __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); - __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); - /* xacc[i] ^= xsecret[i]; */ - __m128i const key_vec = _mm_loadu_si128 (xsecret+i); - __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); - - /* xacc[i] *= XXH_PRIME32_1; */ - __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); - __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); - __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); - xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); - } + XXH_FORCE_INLINE XXH_TARGET_SSE2 + void XXH3_scrambleAcc_sse2(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { + + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + + __m128i *const xacc = (__m128i *)acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i *const xsecret = (const __m128i *)secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { + + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64(acc_vec, 47); + __m128i const data_vec = _mm_xor_si128(acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128(xsecret + i); + __m128i const data_key = _mm_xor_si128(data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = + _mm_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32(data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32(data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + + } + } -XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) -{ - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); - (void)(&XXH_writeLE64); - { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); - -# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 - /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ - XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; - __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); -# else - __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); -# endif - int i; - - const void* const src16 = XXH3_kSecret; - __m128i* dst16 = (__m128i*) customSecret; -# if defined(__GNUC__) || defined(__clang__) - /* - * On GCC & Clang, marking 'dest' as modified will cause the compiler: - * - do not extract the secret from sse registers in the internal loop - * - use less common registers, and avoid pushing these reg into stack - */ - XXH_COMPILER_GUARD(dst16); -# endif - XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ - XXH_ASSERT(((size_t)dst16 & 15) == 0); +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2( + void *XXH_RESTRICT customSecret, xxh_u64 seed64) { + + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { + + int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + + #if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) + const xxh_i64 seed64x2[2] = {(xxh_i64)seed64, (xxh_i64)(0U - seed64)}; + __m128i const seed = _mm_load_si128((__m128i const *)seed64x2); + #else + __m128i const seed = + _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); + #endif + int i; + + const void *const src16 = XXH3_kSecret; + __m128i *dst16 = (__m128i *)customSecret; + #if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); + #endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i = 0; i < nbRounds; ++i) { + + dst16[i] = + _mm_add_epi64(_mm_load_si128((const __m128i *)src16 + i), seed); + + } + + } - for (i=0; i < nbRounds; ++i) { - dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); - } } } -#endif + #endif -#if (XXH_VECTOR == XXH_NEON) + #if (XXH_VECTOR == XXH_NEON) /* forward declarations for the scalar routines */ -XXH_FORCE_INLINE void -XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, - void const* XXH_RESTRICT secret, size_t lane); +XXH_FORCE_INLINE void XXH3_scalarRound(void *XXH_RESTRICT acc, + void const *XXH_RESTRICT input, + void const *XXH_RESTRICT secret, + size_t lane); -XXH_FORCE_INLINE void -XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, - void const* XXH_RESTRICT secret, size_t lane); +XXH_FORCE_INLINE void XXH3_scalarScrambleRound(void *XXH_RESTRICT acc, + void const *XXH_RESTRICT secret, + size_t lane); /*! * @internal @@ -5168,7 +5808,8 @@ XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, * is to optimize the pipelining and can have up to 15% speedup depending on the * CPU, and it also mitigates some GCC codegen issues. * - * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * @see XXH3_NEON_LANES for configuring this and details about this + * optimization. * * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit * integers instead of the other platforms which mask full 64-bit vectors, @@ -5180,740 +5821,866 @@ XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, * there needs to be *three* versions of the accumulate operation used * for the remaining 2 lanes. * - * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap - * nearly perfectly. + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics + * overlap nearly perfectly. */ -XXH_FORCE_INLINE void -XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, - const void* XXH_RESTRICT input, - const void* XXH_RESTRICT secret) -{ - XXH_ASSERT((((size_t)acc) & 15) == 0); - XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); - { /* GCC for darwin arm64 does not like aliasing here */ - xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; - /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ - uint8_t const* xinput = (const uint8_t *) input; - uint8_t const* xsecret = (const uint8_t *) secret; - - size_t i; -#ifdef __wasm_simd128__ - /* - * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret - * is constant propagated, which results in it converting it to this - * inside the loop: - * - * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) - * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) - * ... - * - * This requires a full 32-bit address immediate (and therefore a 6 byte - * instruction) as well as an add for each offset. - * - * Putting an asm guard prevents it from folding (at the cost of losing - * the alignment hint), and uses the free offset in `v128.load` instead - * of adding secret_offset each time which overall reduces code size by - * about a kilobyte and improves performance. - */ - XXH_COMPILER_GUARD(xsecret); -#endif - /* Scalar lanes use the normal scalarRound routine */ - for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { - XXH3_scalarRound(acc, input, secret, i); - } - i = 0; - /* 4 NEON lanes at a time. */ - for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { - /* data_vec = xinput[i]; */ - uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); - uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); - /* key_vec = xsecret[i]; */ - uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); - uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); - /* data_swap = swap(data_vec) */ - uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); - uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); - /* data_key = data_vec ^ key_vec; */ - uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); - uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); - - /* - * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a - * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to - * get one vector with the low 32 bits of each lane, and one vector - * with the high 32 bits of each lane. - * - * The intrinsic returns a double vector because the original ARMv7-a - * instruction modified both arguments in place. AArch64 and SIMD128 emit - * two instructions from this intrinsic. - * - * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] - * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] - */ - uint32x4x2_t unzipped = vuzpq_u32( - vreinterpretq_u32_u64(data_key_1), - vreinterpretq_u32_u64(data_key_2) - ); - /* data_key_lo = data_key & 0xFFFFFFFF */ - uint32x4_t data_key_lo = unzipped.val[0]; - /* data_key_hi = data_key >> 32 */ - uint32x4_t data_key_hi = unzipped.val[1]; - /* - * Then, we can split the vectors horizontally and multiply which, as for most - * widening intrinsics, have a variant that works on both high half vectors - * for free on AArch64. A similar instruction is available on SIMD128. - * - * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi - */ - uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); - uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); - /* - * Clang reorders - * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s - * c += a; // add acc.2d, acc.2d, swap.2d - * to - * c += a; // add acc.2d, acc.2d, swap.2d - * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s - * - * While it would make sense in theory since the addition is faster, - * for reasons likely related to umlal being limited to certain NEON - * pipelines, this is worse. A compiler guard fixes this. - */ - XXH_COMPILER_GUARD_CLANG_NEON(sum_1); - XXH_COMPILER_GUARD_CLANG_NEON(sum_2); - /* xacc[i] = acc_vec + sum; */ - xacc[i] = vaddq_u64(xacc[i], sum_1); - xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); - } - /* Operate on the remaining NEON lanes 2 at a time. */ - for (; i < XXH3_NEON_LANES / 2; i++) { - /* data_vec = xinput[i]; */ - uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); - /* key_vec = xsecret[i]; */ - uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); - /* acc_vec_2 = swap(data_vec) */ - uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); - /* data_key = data_vec ^ key_vec; */ - uint64x2_t data_key = veorq_u64(data_vec, key_vec); - /* For two lanes, just use VMOVN and VSHRN. */ - /* data_key_lo = data_key & 0xFFFFFFFF; */ - uint32x2_t data_key_lo = vmovn_u64(data_key); - /* data_key_hi = data_key >> 32; */ - uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); - /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ - uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); - /* Same Clang workaround as before */ - XXH_COMPILER_GUARD_CLANG_NEON(sum); - /* xacc[i] = acc_vec + sum; */ - xacc[i] = vaddq_u64 (xacc[i], sum); - } +XXH_FORCE_INLINE void XXH3_accumulate_512_neon( + void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, + const void *XXH_RESTRICT secret) { + + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && + XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t *const xacc = (xxh_aliasing_uint64x2_t *)acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. + */ + uint8_t const *xinput = (const uint8_t *)input; + uint8_t const *xsecret = (const uint8_t *)secret; + + size_t i; + #ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); + #endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + + XXH3_scalarRound(acc, input, secret, i); + + } + + i = 0; + /* 4 NEON lanes at a time. */ + for (; i + 1 < XXH3_NEON_LANES / 2; i += 2) { + + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i + 1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i + 1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32(vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2)); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for + * most widening intrinsics, have a variant that works on both high half + * vectors for free on AArch64. A similar instruction is available on + * SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = + XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = + XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i + 1] = vaddq_u64(xacc[i + 1], sum_2); + + } + + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum); + } + + } + } + XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) -XXH_FORCE_INLINE void -XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) -{ - XXH_ASSERT((((size_t)acc) & 15) == 0); - - { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; - uint8_t const* xsecret = (uint8_t const*) secret; - - size_t i; - /* WASM uses operator overloads and doesn't need these. */ -#ifndef __wasm_simd128__ - /* { prime32_1, prime32_1 } */ - uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); - /* { 0, prime32_1, 0, prime32_1 } */ - uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); -#endif + XXH_FORCE_INLINE + void XXH3_scrambleAcc_neon(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { - /* AArch64 uses both scalar and neon at the same time */ - for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { - XXH3_scalarScrambleRound(acc, secret, i); - } - for (i=0; i < XXH3_NEON_LANES / 2; i++) { - /* xacc[i] ^= (xacc[i] >> 47); */ - uint64x2_t acc_vec = xacc[i]; - uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); - uint64x2_t data_vec = veorq_u64(acc_vec, shifted); - - /* xacc[i] ^= xsecret[i]; */ - uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); - uint64x2_t data_key = veorq_u64(data_vec, key_vec); + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { + + xxh_aliasing_uint64x2_t *xacc = (xxh_aliasing_uint64x2_t *)acc; + uint8_t const *xsecret = (uint8_t const *)secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ + #ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = + vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); + #endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + + XXH3_scalarScrambleRound(acc, secret, i); + + } + + for (i = 0; i < XXH3_NEON_LANES / 2; i++) { + + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); /* xacc[i] *= XXH_PRIME32_1 */ -#ifdef __wasm_simd128__ - /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ - xacc[i] = data_key * XXH_PRIME32_1; -#else - /* - * Expanded version with portable NEON intrinsics - * - * lo(x) * lo(y) + (hi(x) * lo(y) << 32) - * - * prod_hi = hi(data_key) * lo(prime) << 32 - * - * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector - * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits - * and avoid the shift. - */ - uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); - /* Extract low bits for vmlal_u32 */ - uint32x2_t data_key_lo = vmovn_u64(data_key); - /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ - xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); -#endif - } + #ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and + * scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; + #else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, + * reinterpreting the vector as a uint32x4_t and multiplying by { 0, + * prime, 0, prime } to cancel out the unwanted bits and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32(vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = + vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); + #endif + } + + } + } -#endif -#if (XXH_VECTOR == XXH_VSX) + #endif + + #if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void XXH3_accumulate_512_vsx(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT input, + const void *XXH_RESTRICT secret) { + + /* presumed aligned */ + xxh_aliasing_u64x2 *const xacc = (xxh_aliasing_u64x2 *)acc; + xxh_u8 const *const xinput = + (xxh_u8 const *)input; /* no alignment restriction */ + xxh_u8 const *const xsecret = + (xxh_u8 const *)secret; /* no alignment restriction */ + xxh_u64x2 const v32 = {32, 32}; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16 * i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16 * i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & + * 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ + #ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); + #else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); + #endif + xacc[i] = acc_vec; + + } + +} + +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + + XXH_FORCE_INLINE + void XXH3_scrambleAcc_vsx(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { + + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { -XXH_FORCE_INLINE void -XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, - const void* XXH_RESTRICT input, - const void* XXH_RESTRICT secret) -{ - /* presumed aligned */ - xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; - xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ - xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ - xxh_u64x2 const v32 = { 32, 32 }; - size_t i; + xxh_aliasing_u64x2 *const xacc = (xxh_aliasing_u64x2 *)acc; + const xxh_u8 *const xsecret = (const xxh_u8 *)secret; + /* constants */ + xxh_u64x2 const v32 = {32, 32}; + xxh_u64x2 const v47 = {47, 47}; + xxh_u32x4 const prime = {XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, + XXH_PRIME32_1}; + size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { - /* data_vec = xinput[i]; */ - xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); - /* key_vec = xsecret[i]; */ - xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); - xxh_u64x2 const data_key = data_vec ^ key_vec; - /* shuffled = (data_key << 32) | (data_key >> 32); */ - xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); - /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ - xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); - /* acc_vec = xacc[i]; */ - xxh_u64x2 acc_vec = xacc[i]; - acc_vec += product; - - /* swap high and low halves */ -#ifdef __s390x__ - acc_vec += vec_permi(data_vec, data_vec, 2); -#else - acc_vec += vec_xxpermdi(data_vec, data_vec, 2); -#endif - xacc[i] = acc_vec; + + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16 * i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & + * 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } + + } + } -XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) -XXH_FORCE_INLINE void -XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) -{ - XXH_ASSERT((((size_t)acc) & 15) == 0); - - { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; - const xxh_u8* const xsecret = (const xxh_u8*) secret; - /* constants */ - xxh_u64x2 const v32 = { 32, 32 }; - xxh_u64x2 const v47 = { 47, 47 }; - xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; - size_t i; - for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { - /* xacc[i] ^= (xacc[i] >> 47); */ - xxh_u64x2 const acc_vec = xacc[i]; - xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); - - /* xacc[i] ^= xsecret[i]; */ - xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); - xxh_u64x2 const data_key = data_vec ^ key_vec; + #endif + + #if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void XXH3_accumulate_512_sve(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT input, + const void *XXH_RESTRICT secret) { + + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + + } else if (element_count == 2) { /* sve128 */ + + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + + } else { + + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + + } - /* xacc[i] *= XXH_PRIME32_1 */ - /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ - xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); - /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ - xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); - xacc[i] = prod_odd + (prod_even << v32); - } } } -#endif +XXH_FORCE_INLINE void XXH3_accumulate_sve(xxh_u64 *XXH_RESTRICT acc, + const xxh_u8 *XXH_RESTRICT input, + const xxh_u8 *XXH_RESTRICT secret, + size_t nbStripes) { -#if (XXH_VECTOR == XXH_SVE) + if (nbStripes != 0) { -XXH_FORCE_INLINE void -XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, - const void* XXH_RESTRICT input, - const void* XXH_RESTRICT secret) -{ - uint64_t *xacc = (uint64_t *)acc; + uint64_t *xacc = (uint64_t *)acc; const uint64_t *xinput = (const uint64_t *)(const void *)input; const uint64_t *xsecret = (const uint64_t *)(const void *)secret; - svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); - uint64_t element_count = svcntd(); + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); if (element_count >= 8) { - svbool_t mask = svptrue_pat_b64(SV_VL8); - svuint64_t vacc = svld1_u64(mask, xacc); + + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); ACCRND(vacc, 0); - svst1_u64(mask, xacc, vacc); - } else if (element_count == 2) { /* sve128 */ - svbool_t mask = svptrue_pat_b64(SV_VL2); - svuint64_t acc0 = svld1_u64(mask, xacc + 0); - svuint64_t acc1 = svld1_u64(mask, xacc + 2); - svuint64_t acc2 = svld1_u64(mask, xacc + 4); - svuint64_t acc3 = svld1_u64(mask, xacc + 6); + xinput += 8; + xsecret += 1; + nbStripes--; + + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + + } else if (element_count == 2) { /* sve128 */ + + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + + svprfd(mask, xinput + 128, SV_PLDL1STRM); ACCRND(acc0, 0); ACCRND(acc1, 2); ACCRND(acc2, 4); ACCRND(acc3, 6); - svst1_u64(mask, xacc + 0, acc0); - svst1_u64(mask, xacc + 2, acc1); - svst1_u64(mask, xacc + 4, acc2); - svst1_u64(mask, xacc + 6, acc3); + xinput += 8; + xsecret += 1; + nbStripes--; + + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { - svbool_t mask = svptrue_pat_b64(SV_VL4); - svuint64_t acc0 = svld1_u64(mask, xacc + 0); - svuint64_t acc1 = svld1_u64(mask, xacc + 4); + + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + + svprfd(mask, xinput + 128, SV_PLDL1STRM); ACCRND(acc0, 0); ACCRND(acc1, 4); - svst1_u64(mask, xacc + 0, acc0); - svst1_u64(mask, xacc + 4, acc1); - } -} + xinput += 8; + xsecret += 1; + nbStripes--; + + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); -XXH_FORCE_INLINE void -XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, - const xxh_u8* XXH_RESTRICT input, - const xxh_u8* XXH_RESTRICT secret, - size_t nbStripes) -{ - if (nbStripes != 0) { - uint64_t *xacc = (uint64_t *)acc; - const uint64_t *xinput = (const uint64_t *)(const void *)input; - const uint64_t *xsecret = (const uint64_t *)(const void *)secret; - svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); - uint64_t element_count = svcntd(); - if (element_count >= 8) { - svbool_t mask = svptrue_pat_b64(SV_VL8); - svuint64_t vacc = svld1_u64(mask, xacc + 0); - do { - /* svprfd(svbool_t, void *, enum svfprop); */ - svprfd(mask, xinput + 128, SV_PLDL1STRM); - ACCRND(vacc, 0); - xinput += 8; - xsecret += 1; - nbStripes--; - } while (nbStripes != 0); - - svst1_u64(mask, xacc + 0, vacc); - } else if (element_count == 2) { /* sve128 */ - svbool_t mask = svptrue_pat_b64(SV_VL2); - svuint64_t acc0 = svld1_u64(mask, xacc + 0); - svuint64_t acc1 = svld1_u64(mask, xacc + 2); - svuint64_t acc2 = svld1_u64(mask, xacc + 4); - svuint64_t acc3 = svld1_u64(mask, xacc + 6); - do { - svprfd(mask, xinput + 128, SV_PLDL1STRM); - ACCRND(acc0, 0); - ACCRND(acc1, 2); - ACCRND(acc2, 4); - ACCRND(acc3, 6); - xinput += 8; - xsecret += 1; - nbStripes--; - } while (nbStripes != 0); - - svst1_u64(mask, xacc + 0, acc0); - svst1_u64(mask, xacc + 2, acc1); - svst1_u64(mask, xacc + 4, acc2); - svst1_u64(mask, xacc + 6, acc3); - } else { - svbool_t mask = svptrue_pat_b64(SV_VL4); - svuint64_t acc0 = svld1_u64(mask, xacc + 0); - svuint64_t acc1 = svld1_u64(mask, xacc + 4); - do { - svprfd(mask, xinput + 128, SV_PLDL1STRM); - ACCRND(acc0, 0); - ACCRND(acc1, 4); - xinput += 8; - xsecret += 1; - nbStripes--; - } while (nbStripes != 0); - - svst1_u64(mask, xacc + 0, acc0); - svst1_u64(mask, xacc + 4, acc1); - } } + + } + } -#endif + #endif -/* scalar variants - universal */ + /* scalar variants - universal */ -#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) + #if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) /* * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they * emit an excess mask and a full 64-bit multiply-add (MADD X-form). * - * While this might not seem like much, as AArch64 is a 64-bit architecture, only - * big Cortex designs have a full 64-bit multiplier. + * While this might not seem like much, as AArch64 is a 64-bit architecture, + * only big Cortex designs have a full 64-bit multiplier. * * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit * multiplies expand to 2-3 multiplies in microcode. This has a major penalty * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. * - * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does - * not have this penalty and does the mask automatically. + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) + * which does not have this penalty and does the mask automatically. */ -XXH_FORCE_INLINE xxh_u64 -XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) -{ - xxh_u64 ret; - /* note: %x = 64-bit register, %w = 32-bit register */ - __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); - return ret; -} -#else -XXH_FORCE_INLINE xxh_u64 -XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) -{ - return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +XXH_FORCE_INLINE xxh_u64 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, + xxh_u64 acc) { + + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" + : "=r"(ret) + : "r"(lhs), "r"(rhs), "r"(acc)); + return ret; + } -#endif + + #else +XXH_FORCE_INLINE xxh_u64 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, + xxh_u64 acc) { + + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; + +} + + #endif /*! * @internal * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). * - * This is extracted to its own function because the NEON path uses a combination - * of NEON and scalar. + * This is extracted to its own function because the NEON path uses a + * combination of NEON and scalar. */ -XXH_FORCE_INLINE void -XXH3_scalarRound(void* XXH_RESTRICT acc, - void const* XXH_RESTRICT input, - void const* XXH_RESTRICT secret, - size_t lane) -{ - xxh_u64* xacc = (xxh_u64*) acc; - xxh_u8 const* xinput = (xxh_u8 const*) input; - xxh_u8 const* xsecret = (xxh_u8 const*) secret; - XXH_ASSERT(lane < XXH_ACC_NB); - XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); - { - xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); - xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); - xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ - xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); - } +XXH_FORCE_INLINE void XXH3_scalarRound(void *XXH_RESTRICT acc, + void const *XXH_RESTRICT input, + void const *XXH_RESTRICT secret, + size_t lane) { + + xxh_u64 *xacc = (xxh_u64 *)acc; + xxh_u8 const *xinput = (xxh_u8 const *)input; + xxh_u8 const *xsecret = (xxh_u8 const *)secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN - 1)) == 0); + { + + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, + data_key >> 32, xacc[lane]); + + } + } /*! * @internal * @brief Processes a 64 byte block of data using the scalar path. */ -XXH_FORCE_INLINE void -XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, - const void* XXH_RESTRICT input, - const void* XXH_RESTRICT secret) -{ - size_t i; - /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ -#if defined(__GNUC__) && !defined(__clang__) \ - && (defined(__arm__) || defined(__thumb2__)) \ - && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ - && XXH_SIZE_OPT <= 0 -# pragma GCC unroll 8 -#endif - for (i=0; i < XXH_ACC_NB; i++) { - XXH3_scalarRound(acc, input, secret, i); - } +XXH_FORCE_INLINE void XXH3_accumulate_512_scalar( + void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, + const void *XXH_RESTRICT secret) { + + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on + * ARMv6. */ + #if defined(__GNUC__) && !defined(__clang__) && \ + (defined(__arm__) || defined(__thumb2__)) && \ + defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes \ + bytes */ \ + && XXH_SIZE_OPT <= 0 + #pragma GCC unroll 8 + #endif + for (i = 0; i < XXH_ACC_NB; i++) { + + XXH3_scalarRound(acc, input, secret, i); + + } + } + XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) -/*! - * @internal - * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). - * - * This is extracted to its own function because the NEON path uses a combination - * of NEON and scalar. - */ -XXH_FORCE_INLINE void -XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, - void const* XXH_RESTRICT secret, - size_t lane) -{ - xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ - const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ - XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); - XXH_ASSERT(lane < XXH_ACC_NB); - { - xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); - xxh_u64 acc64 = xacc[lane]; - acc64 = XXH_xorshift64(acc64, 47); - acc64 ^= key64; - acc64 *= XXH_PRIME32_1; - xacc[lane] = acc64; - } + /*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a + * combination of NEON and scalar. + */ + XXH_FORCE_INLINE + void XXH3_scalarScrambleRound(void *XXH_RESTRICT acc, + void const *XXH_RESTRICT secret, + size_t lane) { + + xxh_u64 *const xacc = (xxh_u64 *)acc; /* presumed aligned */ + const xxh_u8 *const xsecret = + (const xxh_u8 *)secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN - 1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + + } + } /*! * @internal * @brief Scrambles the accumulators after a large chunk has been read */ -XXH_FORCE_INLINE void -XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) -{ - size_t i; - for (i=0; i < XXH_ACC_NB; i++) { - XXH3_scalarScrambleRound(acc, secret, i); +XXH_FORCE_INLINE void XXH3_scrambleAcc_scalar(void *XXH_RESTRICT acc, + const void *XXH_RESTRICT secret) { + + size_t i; + for (i = 0; i < XXH_ACC_NB; i++) { + + XXH3_scalarScrambleRound(acc, secret, i); + + } + +} + +XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( + void *XXH_RESTRICT customSecret, xxh_u64 seed64) { + + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8 *kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + + #if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); + #endif + { + + int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i = 0; i < nbRounds; i++) { + + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16 * i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16 * i + 8) - seed64; + XXH_writeLE64((xxh_u8 *)customSecret + 16 * i, lo); + XXH_writeLE64((xxh_u8 *)customSecret + 16 * i + 8, hi); + } -} -XXH_FORCE_INLINE void -XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) -{ - /* - * We need a separate pointer for the hack below, - * which requires a non-const pointer. - * Any decent compiler will optimize this out otherwise. - */ - const xxh_u8* kSecretPtr = XXH3_kSecret; - XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + } -#if defined(__GNUC__) && defined(__aarch64__) - /* - * UGLY HACK: - * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are - * placed sequentially, in order, at the top of the unrolled loop. - * - * While MOVK is great for generating constants (2 cycles for a 64-bit - * constant compared to 4 cycles for LDR), it fights for bandwidth with - * the arithmetic instructions. - * - * I L S - * MOVK - * MOVK - * MOVK - * MOVK - * ADD - * SUB STR - * STR - * By forcing loads from memory (as the asm line causes the compiler to assume - * that XXH3_kSecretPtr has been changed), the pipelines are used more - * efficiently: - * I L S - * LDR - * ADD LDR - * SUB STR - * STR - * - * See XXH3_NEON_LANES for details on the pipsline. - * - * XXH3_64bits_withSeed, len == 256, Snapdragon 835 - * without hack: 2654.4 MB/s - * with hack: 3202.9 MB/s - */ - XXH_COMPILER_GUARD(kSecretPtr); -#endif - { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; - int i; - for (i=0; i < nbRounds; i++) { - /* - * The asm hack causes the compiler to assume that kSecretPtr aliases with - * customSecret, and on aarch64, this prevented LDP from merging two - * loads together for free. Putting the loads together before the stores - * properly generates LDP. - */ - xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; - xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; - XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); - XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); - } } } +typedef void (*XXH3_f_accumulate)(xxh_u64 *XXH_RESTRICT, + const xxh_u8 *XXH_RESTRICT, + const xxh_u8 *XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void *XXH_RESTRICT, const void *); +typedef void (*XXH3_f_initCustomSecret)(void *XXH_RESTRICT, xxh_u64); -typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); -typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); -typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + #if (XXH_VECTOR == XXH_AVX512) + #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 + #define XXH3_accumulate XXH3_accumulate_avx512 + #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 + #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 -#if (XXH_VECTOR == XXH_AVX512) + #elif (XXH_VECTOR == XXH_AVX2) -#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 -#define XXH3_accumulate XXH3_accumulate_avx512 -#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 -#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 + #define XXH3_accumulate XXH3_accumulate_avx2 + #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 + #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 -#elif (XXH_VECTOR == XXH_AVX2) + #elif (XXH_VECTOR == XXH_SSE2) -#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 -#define XXH3_accumulate XXH3_accumulate_avx2 -#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 -#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 + #define XXH3_accumulate XXH3_accumulate_sse2 + #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 + #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 -#elif (XXH_VECTOR == XXH_SSE2) + #elif (XXH_VECTOR == XXH_NEON) -#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 -#define XXH3_accumulate XXH3_accumulate_sse2 -#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 -#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + #define XXH3_accumulate_512 XXH3_accumulate_512_neon + #define XXH3_accumulate XXH3_accumulate_neon + #define XXH3_scrambleAcc XXH3_scrambleAcc_neon + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar -#elif (XXH_VECTOR == XXH_NEON) + #elif (XXH_VECTOR == XXH_VSX) -#define XXH3_accumulate_512 XXH3_accumulate_512_neon -#define XXH3_accumulate XXH3_accumulate_neon -#define XXH3_scrambleAcc XXH3_scrambleAcc_neon -#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #define XXH3_accumulate_512 XXH3_accumulate_512_vsx + #define XXH3_accumulate XXH3_accumulate_vsx + #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar -#elif (XXH_VECTOR == XXH_VSX) + #elif (XXH_VECTOR == XXH_SVE) + #define XXH3_accumulate_512 XXH3_accumulate_512_sve + #define XXH3_accumulate XXH3_accumulate_sve + #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar -#define XXH3_accumulate_512 XXH3_accumulate_512_vsx -#define XXH3_accumulate XXH3_accumulate_vsx -#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx -#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #else /* scalar */ -#elif (XXH_VECTOR == XXH_SVE) -#define XXH3_accumulate_512 XXH3_accumulate_512_sve -#define XXH3_accumulate XXH3_accumulate_sve -#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar -#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #define XXH3_accumulate_512 XXH3_accumulate_512_scalar + #define XXH3_accumulate XXH3_accumulate_scalar + #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar -#else /* scalar */ + #endif -#define XXH3_accumulate_512 XXH3_accumulate_512_scalar -#define XXH3_accumulate XXH3_accumulate_scalar -#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar -#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ + #undef XXH3_initCustomSecret + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #endif -#endif +XXH_FORCE_INLINE void XXH3_hashLong_internal_loop( + xxh_u64 *XXH_RESTRICT acc, const xxh_u8 *XXH_RESTRICT input, size_t len, + const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { -#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ -# undef XXH3_initCustomSecret -# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar -#endif + size_t const nbStripesPerBlock = + (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; -XXH_FORCE_INLINE void -XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, - const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH3_f_accumulate f_acc, - XXH3_f_scrambleAcc f_scramble) -{ - size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; - size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; - size_t const nb_blocks = (len - 1) / block_len; + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + + f_acc(acc, input + n * block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + + } - size_t n; + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + size_t const nbStripes = + ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks * block_len, secret, nbStripes); + + /* last stripe */ + { + + const xxh_u8 *const p = input + len - XXH_STRIPE_LEN; + #define XXH_SECRET_LASTACC_START \ + 7 /* not aligned on 8, last secret is different from acc & scrambler \ + */ + XXH3_accumulate_512( + acc, p, + secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); - for (n = 0; n < nb_blocks; n++) { - f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); - f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); } - /* last partial block */ - XXH_ASSERT(len > XXH_STRIPE_LEN); - { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; - XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); - f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + } - /* last stripe */ - { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; -#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ - XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); - } } } -XXH_FORCE_INLINE xxh_u64 -XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) -{ - return XXH3_mul128_fold64( - acc[0] ^ XXH_readLE64(secret), - acc[1] ^ XXH_readLE64(secret+8) ); +XXH_FORCE_INLINE xxh_u64 XXH3_mix2Accs(const xxh_u64 *XXH_RESTRICT acc, + const xxh_u8 *XXH_RESTRICT secret) { + + return XXH3_mul128_fold64(acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret + 8)); + } -static XXH64_hash_t -XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) -{ - xxh_u64 result64 = start; - size_t i = 0; - - for (i = 0; i < 4; i++) { - result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); -#if defined(__clang__) /* Clang */ \ - && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ - /* - * UGLY HACK: - * Prevent autovectorization on Clang ARMv7-a. Exact same problem as - * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. - * XXH3_64bits, len == 256, Snapdragon 835: - * without hack: 2063.7 MB/s - * with hack: 2560.7 MB/s - */ - XXH_COMPILER_GUARD(result64); -#endif - } +static XXH64_hash_t XXH3_mergeAccs(const xxh_u64 *XXH_RESTRICT acc, + const xxh_u8 *XXH_RESTRICT secret, + xxh_u64 start) { + + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + + result64 += XXH3_mix2Accs(acc + 2 * i, secret + 16 * i); + #if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); + #endif + + } + + return XXH3_avalanche(result64); - return XXH3_avalanche(result64); } -#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ - XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + #define XXH3_INIT_ACC \ + { \ + \ + XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 \ + \ + } -XXH_FORCE_INLINE XXH64_hash_t -XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, - const void* XXH_RESTRICT secret, size_t secretSize, - XXH3_f_accumulate f_acc, - XXH3_f_scrambleAcc f_scramble) -{ - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; +XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal( + const void *XXH_RESTRICT input, size_t len, const void *XXH_RESTRICT secret, + size_t secretSize, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { + + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; - XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + XXH3_hashLong_internal_loop(acc, (const xxh_u8 *)input, len, + (const xxh_u8 *)secret, secretSize, f_acc, + f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator + */ + #define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, + (const xxh_u8 *)secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); - /* converge into final hash */ - XXH_STATIC_ASSERT(sizeof(acc) == 64); - /* do not align on 8, so that the secret is different from the accumulator */ -#define XXH_SECRET_MERGEACCS_START 11 - XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); - return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); } /* * It's important for performance to transmit secret's size (when it's static) * so that the compiler can properly optimize the vectorized loop. - * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. - * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE - * breaks -Og, this is XXH_NO_INLINE. + * This makes a big performance difference for "medium" keys (<1 KB) when using + * AVX instruction set. When the secret size is unknown, or on GCC 12 where the + * mix of NO_INLINE and FORCE_INLINE breaks -Og, this is XXH_NO_INLINE. */ -XXH3_WITH_SECRET_INLINE XXH64_hash_t -XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, - XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) -{ - (void)seed64; - return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +XXH3_WITH_SECRET_INLINE XXH64_hash_t XXH3_hashLong_64b_withSecret( + const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, + const xxh_u8 *XXH_RESTRICT secret, size_t secretLen) { + + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); + } /* * It's preferable for performance that XXH3_hashLong is not inlined, - * as it results in a smaller function for small data, easier to the instruction cache. - * Note that inside this no_inline function, we do inline the internal loop, - * and provide a statically defined secret size to allow optimization of vector loop. + * as it results in a smaller function for small data, easier to the instruction + * cache. Note that inside this no_inline function, we do inline the internal + * loop, and provide a statically defined secret size to allow optimization of + * vector loop. */ -XXH_NO_INLINE XXH_PUREF XXH64_hash_t -XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, - XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) -{ - (void)seed64; (void)secret; (void)secretLen; - return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +XXH_NO_INLINE XXH_PUREF XXH64_hash_t XXH3_hashLong_64b_default( + const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, + const xxh_u8 *XXH_RESTRICT secret, size_t secretLen) { + + (void)seed64; + (void)secret; + (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, + sizeof(XXH3_kSecret), XXH3_accumulate, + XXH3_scrambleAcc); + } /* * XXH3_hashLong_64b_withSeed(): - * Generate a custom key based on alteration of default XXH3_kSecret with the seed, - * and then use this key for long mode hashing. + * Generate a custom key based on alteration of default XXH3_kSecret with the + * seed, and then use this key for long mode hashing. * * This operation is decently fast but nonetheless costs a little bit of time. * Try to avoid it whenever possible (typically when seed==0). @@ -5921,98 +6688,116 @@ XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, * It's important for performance that XXH3_hashLong is not inlined. Not sure * why (uop cache maybe?), but the difference is large and easily measurable. */ -XXH_FORCE_INLINE XXH64_hash_t -XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, - XXH64_hash_t seed, - XXH3_f_accumulate f_acc, - XXH3_f_scrambleAcc f_scramble, - XXH3_f_initCustomSecret f_initSec) -{ -#if XXH_SIZE_OPT <= 0 - if (seed == 0) - return XXH3_hashLong_64b_internal(input, len, - XXH3_kSecret, sizeof(XXH3_kSecret), - f_acc, f_scramble); -#endif - { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; - f_initSec(secret, seed); - return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), - f_acc, f_scramble); - } +XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed_internal( + const void *input, size_t len, XXH64_hash_t seed, XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, XXH3_f_initCustomSecret f_initSec) { + + #if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, + sizeof(XXH3_kSecret), f_acc, f_scramble); + #endif + { + + XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), f_acc, + f_scramble); + + } + } /* * It's important for performance that XXH3_hashLong is not inlined. */ -XXH_NO_INLINE XXH64_hash_t -XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, - XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) -{ - (void)secret; (void)secretLen; - return XXH3_hashLong_64b_withSeed_internal(input, len, seed, - XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); -} +XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed( + const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed, + const xxh_u8 *XXH_RESTRICT secret, size_t secretLen) { + (void)secret; + (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, XXH3_accumulate, + XXH3_scrambleAcc, + XXH3_initCustomSecret); + +} -typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, - XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void *XXH_RESTRICT, size_t, + XXH64_hash_t, + const xxh_u8 *XXH_RESTRICT, size_t); XXH_FORCE_INLINE XXH64_hash_t -XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, - XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, - XXH3_hashLong64_f f_hashLong) -{ - XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); - /* - * If an action is to be taken if `secretLen` condition is not respected, - * it should be done here. - * For now, it's a contract pre-condition. - * Adding a check and a branch here would cost performance at every hash. - * Also, note that function signature doesn't offer room to return an error. - */ - if (len <= 16) - return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); - if (len <= 128) - return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); - return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); -} +XXH3_64bits_internal(const void *XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void *XXH_RESTRICT secret, + size_t secretLen, XXH3_hashLong64_f f_hashLong) { + + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8 *)input, len, + (const xxh_u8 *)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8 *)input, len, + (const xxh_u8 *)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8 *)input, len, + (const xxh_u8 *)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8 *)secret, secretLen); +} /* === Public entry point === */ /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) -{ - return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void *input, + size_t length) { + + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, + sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); + } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH64_hash_t -XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) -{ - return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +XXH3_64bits_withSecret(XXH_NOESCAPE const void *input, size_t length, + XXH_NOESCAPE const void *secret, size_t secretSize) { + + return XXH3_64bits_internal(input, length, 0, secret, secretSize, + XXH3_hashLong_64b_withSecret); + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH64_hash_t -XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) -{ - return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); -} +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void *input, + size_t length, + XXH64_hash_t seed) { + + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, + sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); -XXH_PUBLIC_API XXH64_hash_t -XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) -{ - if (length <= XXH3_MIDSIZE_MAX) - return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); - return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); } +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecretandSeed( + XXH_NOESCAPE const void *input, size_t length, + XXH_NOESCAPE const void *secret, size_t secretSize, XXH64_hash_t seed) { + + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, + sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, + (const xxh_u8 *)secret, secretSize); + +} -/* === XXH3 streaming === */ -#ifndef XXH_NO_STREAM + /* === XXH3 streaming === */ + #ifndef XXH_NO_STREAM /* * Malloc's a pointer that is always aligned to align. * @@ -6036,48 +6821,58 @@ XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH * * Align must be a power of 2 and 8 <= align <= 128. */ -static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) -{ - XXH_ASSERT(align <= 128 && align >= 8); /* range check */ - XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ - XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ - { /* Overallocate to make room for manual realignment and an offset byte */ - xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); - if (base != NULL) { - /* - * Get the offset needed to align this pointer. - * - * Even if the returned pointer is aligned, there will always be - * at least one byte to store the offset to the original pointer. - */ - size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ - /* Add the offset for the now-aligned pointer */ - xxh_u8* ptr = base + offset; - - XXH_ASSERT((size_t)ptr % align == 0); - - /* Store the offset immediately before the returned pointer. */ - ptr[-1] = (xxh_u8)offset; - return ptr; - } - return NULL; +static XXH_MALLOCF void *XXH_alignedMalloc(size_t s, size_t align) { + + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align - 1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8 *base = (xxh_u8 *)XXH_malloc(s + align); + if (base != NULL) { + + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8 *ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + + return NULL; + + } + } + /* * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. */ -static void XXH_alignedFree(void* p) -{ - if (p != NULL) { - xxh_u8* ptr = (xxh_u8*)p; - /* Get the offset byte we added in XXH_malloc. */ - xxh_u8 offset = ptr[-1]; - /* Free the original malloc'd pointer */ - xxh_u8* base = ptr - offset; - XXH_free(base); - } +static void XXH_alignedFree(void *p) { + + if (p != NULL) { + + xxh_u8 *ptr = (xxh_u8 *)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8 *base = ptr - offset; + XXH_free(base); + + } + } + /*! @ingroup XXH3_family */ /*! * @brief Allocate an @ref XXH3_state_t. @@ -6089,19 +6884,22 @@ static void XXH_alignedFree(void* p) * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) -{ - XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); - if (state==NULL) return NULL; - XXH3_INITSTATE(state); - return state; +XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void) { + + XXH3_state_t *const state = + (XXH3_state_t *)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state == NULL) return NULL; + XXH3_INITSTATE(state); + return state; + } /*! @ingroup XXH3_family */ /*! * @brief Frees an @ref XXH3_state_t. * - * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref + * XXH3_createState(). * * @return @ref XXH_OK. * @@ -6109,98 +6907,108 @@ XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) * * @see @ref streaming_example "Streaming Example" */ -XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) -{ - XXH_alignedFree(statePtr); - return XXH_OK; +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr) { + + XXH_alignedFree(statePtr); + return XXH_OK; + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API void -XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) -{ - XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t *dst_state, + XXH_NOESCAPE const XXH3_state_t *src_state) { + + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); + } -static void -XXH3_reset_internal(XXH3_state_t* statePtr, - XXH64_hash_t seed, - const void* secret, size_t secretSize) -{ - size_t const initStart = offsetof(XXH3_state_t, bufferedSize); - size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; - XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); - XXH_ASSERT(statePtr != NULL); - /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ - memset((char*)statePtr + initStart, 0, initLength); - statePtr->acc[0] = XXH_PRIME32_3; - statePtr->acc[1] = XXH_PRIME64_1; - statePtr->acc[2] = XXH_PRIME64_2; - statePtr->acc[3] = XXH_PRIME64_3; - statePtr->acc[4] = XXH_PRIME64_4; - statePtr->acc[5] = XXH_PRIME32_2; - statePtr->acc[6] = XXH_PRIME64_5; - statePtr->acc[7] = XXH_PRIME32_1; - statePtr->seed = seed; - statePtr->useSeed = (seed != 0); - statePtr->extSecret = (const unsigned char*)secret; - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); - statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; - statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +static void XXH3_reset_internal(XXH3_state_t *statePtr, XXH64_hash_t seed, + const void *secret, size_t secretSize) { + + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = + offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char *)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char *)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; + } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) -{ - if (statePtr == NULL) return XXH_ERROR; - XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t *statePtr) { + + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) -{ - if (statePtr == NULL) return XXH_ERROR; - XXH3_reset_internal(statePtr, 0, secret, secretSize); - if (secret == NULL) return XXH_ERROR; - if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; - return XXH_OK; +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize) { + + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) -{ - if (statePtr == NULL) return XXH_ERROR; - if (seed==0) return XXH3_64bits_reset(statePtr); - if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) - XXH3_initCustomSecret(statePtr->customSecret, seed); - XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); - return XXH_OK; +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH64_hash_t seed) { + + if (statePtr == NULL) return XXH_ERROR; + if (seed == 0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) -{ - if (statePtr == NULL) return XXH_ERROR; - if (secret == NULL) return XXH_ERROR; - if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; - XXH3_reset_internal(statePtr, seed64, secret, secretSize); - statePtr->useSeed = 1; /* always, even if seed64==0 */ - return XXH_OK; +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecretandSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed64) { + + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; + } /*! * @internal * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). * - * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a + * block. * * @param acc Pointer to the 8 accumulator lanes - * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in + * the block* * @param nbStripesPerBlock Number of stripes in a block * @param input Input pointer * @param nbStripes Number of stripes to process @@ -6210,200 +7018,233 @@ XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOE * @param f_scramble Pointer to an XXH3_scrambleAcc implementation * @return Pointer past the end of @p input after processing */ -XXH_FORCE_INLINE const xxh_u8 * -XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, - size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, - const xxh_u8* XXH_RESTRICT input, size_t nbStripes, - const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, - XXH3_f_accumulate f_acc, - XXH3_f_scrambleAcc f_scramble) -{ - const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; - /* Process full blocks */ - if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { - /* Process the initial partial block... */ - size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; - - do { - /* Accumulate and scramble */ - f_acc(acc, input, initialSecret, nbStripesThisIter); - f_scramble(acc, secret + secretLimit); - input += nbStripesThisIter * XXH_STRIPE_LEN; - nbStripes -= nbStripesThisIter; - /* Then continue the loop with the full block size */ - nbStripesThisIter = nbStripesPerBlock; - initialSecret = secret; - } while (nbStripes >= nbStripesPerBlock); - *nbStripesSoFarPtr = 0; - } - /* Process a partial block */ - if (nbStripes > 0) { - f_acc(acc, input, initialSecret, nbStripes); - input += nbStripes * XXH_STRIPE_LEN; - *nbStripesSoFarPtr += nbStripes; - } - /* Return end pointer */ - return input; +XXH_FORCE_INLINE const xxh_u8 *XXH3_consumeStripes( + xxh_u64 *XXH_RESTRICT acc, size_t *XXH_RESTRICT nbStripesSoFarPtr, + size_t nbStripesPerBlock, const xxh_u8 *XXH_RESTRICT input, + size_t nbStripes, const xxh_u8 *XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { + + const xxh_u8 *initialSecret = + secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + + } while (nbStripes >= nbStripesPerBlock); + + *nbStripesSoFarPtr = 0; + + } + + /* Process a partial block */ + if (nbStripes > 0) { + + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + + } + + /* Return end pointer */ + return input; + } -#ifndef XXH3_STREAM_USE_STACK -# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ -# define XXH3_STREAM_USE_STACK 1 -# endif -#endif + #ifndef XXH3_STREAM_USE_STACK + #if XXH_SIZE_OPT <= 0 && \ + !defined( \ + __clang__) /* clang doesn't need additional stack space */ + #define XXH3_STREAM_USE_STACK 1 + #endif + #endif /* * Both XXH3_64bits_update and XXH3_128bits_update use this routine. */ -XXH_FORCE_INLINE XXH_errorcode -XXH3_update(XXH3_state_t* XXH_RESTRICT const state, - const xxh_u8* XXH_RESTRICT input, size_t len, - XXH3_f_accumulate f_acc, - XXH3_f_scrambleAcc f_scramble) -{ - if (input==NULL) { - XXH_ASSERT(len == 0); - return XXH_OK; - } +XXH_FORCE_INLINE XXH_errorcode XXH3_update( + XXH3_state_t *XXH_RESTRICT const state, const xxh_u8 *XXH_RESTRICT input, + size_t len, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { - XXH_ASSERT(state != NULL); - { const xxh_u8* const bEnd = input + len; - const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; -#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 - /* For some reason, gcc and MSVC seem to suffer greatly - * when operating accumulators directly into state. - * Operating into stack space seems to enable proper optimization. - * clang, on the other hand, doesn't seem to need this trick */ - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; - XXH_memcpy(acc, state->acc, sizeof(acc)); -#else - xxh_u64* XXH_RESTRICT const acc = state->acc; -#endif - state->totalLen += len; - XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); - - /* small input : just fill in tmp buffer */ - if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { - XXH_memcpy(state->buffer + state->bufferedSize, input, len); - state->bufferedSize += (XXH32_hash_t)len; - return XXH_OK; - } + if (input == NULL) { + + XXH_ASSERT(len == 0); + return XXH_OK; + + } + + XXH_ASSERT(state != NULL); + { + + const xxh_u8 *const bEnd = input + len; + const unsigned char *const secret = + (state->extSecret == NULL) ? state->customSecret : state->extSecret; + #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); + #else + xxh_u64 *XXH_RESTRICT const acc = state->acc; + #endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + + } /* total input is now > XXH3_INTERNALBUFFER_SIZE */ - #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) - XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + #define XXH3_INTERNALBUFFER_STRIPES \ + (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == + 0); /* clean multiple */ - /* - * Internal buffer is partially filled (always, except at beginning) - * Complete it, then consume it. - */ - if (state->bufferedSize) { - size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; - XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); - input += loadSize; - XXH3_consumeStripes(acc, - &state->nbStripesSoFar, state->nbStripesPerBlock, - state->buffer, XXH3_INTERNALBUFFER_STRIPES, - secret, state->secretLimit, - f_acc, f_scramble); - state->bufferedSize = 0; - } - XXH_ASSERT(input < bEnd); - if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { - size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; - input = XXH3_consumeStripes(acc, - &state->nbStripesSoFar, state->nbStripesPerBlock, - input, nbStripes, - secret, state->secretLimit, - f_acc, f_scramble); - XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, secret, + state->secretLimit, f_acc, f_scramble); + state->bufferedSize = 0; + + } + + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes( + acc, &state->nbStripesSoFar, state->nbStripesPerBlock, input, + nbStripes, secret, state->secretLimit, f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, + input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); - } - /* Some remaining input (always) : buffer it */ - XXH_ASSERT(input < bEnd); - XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); - XXH_ASSERT(state->bufferedSize == 0); - XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); - state->bufferedSize = (XXH32_hash_t)(bEnd-input); -#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 - /* save stack accumulators into state */ - XXH_memcpy(state->acc, acc, sizeof(acc)); -#endif } - return XXH_OK; + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd - input)); + state->bufferedSize = (XXH32_hash_t)(bEnd - input); + #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); + #endif + + } + + return XXH_OK; + } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode -XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) -{ - return XXH3_update(state, (const xxh_u8*)input, len, - XXH3_accumulate, XXH3_scrambleAcc); +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t *state, + XXH_NOESCAPE const void *input, size_t len) { + + return XXH3_update(state, (const xxh_u8 *)input, len, XXH3_accumulate, + XXH3_scrambleAcc); + } +XXH_FORCE_INLINE void XXH3_digest_long(XXH64_hash_t *acc, + const XXH3_state_t *state, + const unsigned char *secret) { -XXH_FORCE_INLINE void -XXH3_digest_long (XXH64_hash_t* acc, - const XXH3_state_t* state, - const unsigned char* secret) -{ - xxh_u8 lastStripe[XXH_STRIPE_LEN]; - const xxh_u8* lastStripePtr; + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8 *lastStripePtr; + + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + + } else { /* bufferedSize < XXH_STRIPE_LEN */ + + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > + 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, + catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + + } + + /* Last stripe */ + XXH3_accumulate_512(acc, lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); - /* - * Digest on a local copy. This way, the state remains unaltered, and it can - * continue ingesting more input afterwards. - */ - XXH_memcpy(acc, state->acc, sizeof(state->acc)); - if (state->bufferedSize >= XXH_STRIPE_LEN) { - /* Consume remaining stripes then point to remaining data in buffer */ - size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; - size_t nbStripesSoFar = state->nbStripesSoFar; - XXH3_consumeStripes(acc, - &nbStripesSoFar, state->nbStripesPerBlock, - state->buffer, nbStripes, - secret, state->secretLimit, - XXH3_accumulate, XXH3_scrambleAcc); - lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; - } else { /* bufferedSize < XXH_STRIPE_LEN */ - /* Copy to temp buffer */ - size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; - XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ - XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); - XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); - lastStripePtr = lastStripe; - } - /* Last stripe */ - XXH3_accumulate_512(acc, - lastStripePtr, - secret + state->secretLimit - XXH_SECRET_LASTACC_START); } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) -{ - const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; - if (state->totalLen > XXH3_MIDSIZE_MAX) { - XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; - XXH3_digest_long(acc, state, secret); - return XXH3_mergeAccs(acc, - secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)state->totalLen * XXH_PRIME64_1); - } - /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ - if (state->useSeed) - return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); - return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), - secret, state->secretLimit + XXH_STRIPE_LEN); +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_digest(XXH_NOESCAPE const XXH3_state_t *state) { + + const unsigned char *const secret = + (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + + } + + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, + state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); + } -#endif /* !XXH_NO_STREAM */ + #endif /* !XXH_NO_STREAM */ /* ========================================== * XXH3 128 bits (a.k.a XXH128) * ========================================== - * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, - * even without counting the significantly larger output size. + * XXH3's 128-bit variant has better mixing and strength than the 64-bit + * variant, even without counting the significantly larger output size. * * For example, extra steps are taken to avoid the seed-dependent collisions * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). @@ -6416,503 +7257,614 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). */ -XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t -XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - /* A doubled version of 1to3_64b with different constants. */ - XXH_ASSERT(input != NULL); - XXH_ASSERT(1 <= len && len <= 3); - XXH_ASSERT(secret != NULL); - /* - * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } - * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } - * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_1to3_128b( + const xxh_u8 *input, size_t len, const xxh_u8 *secret, XXH64_hash_t seed) { + + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { + + xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) | + ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = + (XXH_readLE32(secret) ^ XXH_readLE32(secret + 4)) + seed; + xxh_u64 const bitfliph = + (XXH_readLE32(secret + 8) ^ XXH_readLE32(secret + 12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + + } + +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_4to8_128b( + const xxh_u8 *input, size_t len, const xxh_u8 *secret, XXH64_hash_t seed) { + + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { + + xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = + (XXH_readLE64(secret + 16) ^ XXH_readLE64(secret + 24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ - { xxh_u8 const c1 = input[0]; - xxh_u8 const c2 = input[len >> 1]; - xxh_u8 const c3 = input[len - 1]; - xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) - | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); - xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); - xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; - xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; - xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; - xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; - XXH128_hash_t h128; - h128.low64 = XXH64_avalanche(keyed_lo); - h128.high64 = XXH64_avalanche(keyed_hi); - return h128; - } + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + + } + } -XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t -XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(4 <= len && len <= 8); - seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; - { xxh_u32 const input_lo = XXH_readLE32(input); - xxh_u32 const input_hi = XXH_readLE32(input + len - 4); - xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); - xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; - xxh_u64 const keyed = input_64 ^ bitflip; - - /* Shift len to the left to ensure it is even, this avoids even multiplies. */ - XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); - - m128.high64 += (m128.low64 << 1); - m128.low64 ^= (m128.high64 >> 3); - - m128.low64 = XXH_xorshift64(m128.low64, 35); - m128.low64 *= PRIME_MX2; - m128.low64 = XXH_xorshift64(m128.low64, 28); - m128.high64 = XXH3_avalanche(m128.high64); - return m128; +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_9to16_128b( + const xxh_u8 *input, size_t len, const xxh_u8 *secret, XXH64_hash_t seed) { + + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { + + xxh_u64 const bitflipl = + (XXH_readLE64(secret + 32) ^ XXH_readLE64(secret + 40)) - seed; + xxh_u64 const bitfliph = + (XXH_readLE64(secret + 48) ^ XXH_readLE64(secret + 56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = + XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + + } else { + + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - + * 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += + input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } -} -XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t -XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(input != NULL); - XXH_ASSERT(secret != NULL); - XXH_ASSERT(9 <= len && len <= 16); - { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; - xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; - xxh_u64 const input_lo = XXH_readLE64(input); - xxh_u64 input_hi = XXH_readLE64(input + len - 8); - XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); - /* - * Put len in the middle of m128 to ensure that the length gets mixed to - * both the low and high bits in the 128x64 multiply below. - */ - m128.low64 += (xxh_u64)(len - 1) << 54; - input_hi ^= bitfliph; - /* - * Add the high 32 bits of input_hi to the high 32 bits of m128, then - * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to - * the high 64 bits of m128. - * - * The best approach to this operation is different on 32-bit and 64-bit. - */ - if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ - /* - * 32-bit optimized version, which is more readable. - * - * On 32-bit, it removes an ADC and delays a dependency between the two - * halves of m128.high64, but it generates an extra mask on 64-bit. - */ - m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); - } else { - /* - * 64-bit optimized (albeit more confusing) version. - * - * Uses some properties of addition and multiplication to remove the mask: - * - * Let: - * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) - * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) - * c = XXH_PRIME32_2 - * - * a + (b * c) - * Inverse Property: x + y - x == y - * a + (b * (1 + c - 1)) - * Distributive Property: x * (y + z) == (x * y) + (x * z) - * a + (b * 1) + (b * (c - 1)) - * Identity Property: x * 1 == x - * a + b + (b * (c - 1)) - * - * Substitute a, b, and c: - * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) - * - * Since input_hi.hi + input_hi.lo == input_hi, we get this: - * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) - */ - m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); - } - /* m128 ^= XXH_swap64(m128 >> 64); */ - m128.low64 ^= XXH_swap64(m128.high64); + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; - { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ - XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); - h128.high64 += m128.high64 * XXH_PRIME64_2; + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + + } + + } - h128.low64 = XXH3_avalanche(h128.low64); - h128.high64 = XXH3_avalanche(h128.high64); - return h128; - } } } /* * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN */ -XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t -XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) -{ - XXH_ASSERT(len <= 16); - { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); - if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); - if (len) return XXH3_len_1to3_128b(input, len, secret, seed); - { XXH128_hash_t h128; - xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); - xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); - h128.low64 = XXH64_avalanche(seed ^ bitflipl); - h128.high64 = XXH64_avalanche( seed ^ bitfliph); - return h128; - } } +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_0to16_128b( + const xxh_u8 *input, size_t len, const xxh_u8 *secret, XXH64_hash_t seed) { + + XXH_ASSERT(len <= 16); + { + + if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { + + XXH128_hash_t h128; + xxh_u64 const bitflipl = + XXH_readLE64(secret + 64) ^ XXH_readLE64(secret + 72); + xxh_u64 const bitfliph = + XXH_readLE64(secret + 80) ^ XXH_readLE64(secret + 88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche(seed ^ bitfliph); + return h128; + + } + + } + } /* * A bit slower than XXH3_mix16B, but handles multiply by zero better. */ -XXH_FORCE_INLINE XXH128_hash_t -XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, - const xxh_u8* secret, XXH64_hash_t seed) -{ - acc.low64 += XXH3_mix16B (input_1, secret+0, seed); - acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); - acc.high64 += XXH3_mix16B (input_2, secret+16, seed); - acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); - return acc; +XXH_FORCE_INLINE XXH128_hash_t XXH128_mix32B(XXH128_hash_t acc, + const xxh_u8 *input_1, + const xxh_u8 *input_2, + const xxh_u8 *secret, + XXH64_hash_t seed) { + + acc.low64 += XXH3_mix16B(input_1, secret + 0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B(input_2, secret + 16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; + } +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_17to128_128b( + const xxh_u8 *XXH_RESTRICT input, size_t len, + const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { + + XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + + #if XXH_SIZE_OPT >= 1 + { + + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + + acc = XXH128_mix32B(acc, input + 16 * i, input + len - 16 * (i + 1), + secret + 32 * i, seed); + + } while (i-- != 0); -XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t -XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH64_hash_t seed) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; - XXH_ASSERT(16 < len && len <= 128); - - { XXH128_hash_t acc; - acc.low64 = len * XXH_PRIME64_1; - acc.high64 = 0; - -#if XXH_SIZE_OPT >= 1 - { - /* Smaller, but slightly slower. */ - unsigned int i = (unsigned int)(len - 1) / 32; - do { - acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); - } while (i-- != 0); - } -#else - if (len > 32) { - if (len > 64) { - if (len > 96) { - acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); - } - acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); - } - acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); - } - acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); -#endif - { XXH128_hash_t h128; - h128.low64 = acc.low64 + acc.high64; - h128.high64 = (acc.low64 * XXH_PRIME64_1) - + (acc.high64 * XXH_PRIME64_4) - + ((len - seed) * XXH_PRIME64_2); - h128.low64 = XXH3_avalanche(h128.low64); - h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); - return h128; - } } -} -XXH_NO_INLINE XXH_PUREF XXH128_hash_t -XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH64_hash_t seed) -{ - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; - XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + #else + if (len > 32) { + + if (len > 64) { + + if (len > 96) { + + acc = XXH128_mix32B(acc, input + 48, input + len - 64, secret + 96, + seed); - { XXH128_hash_t acc; - unsigned i; - acc.low64 = len * XXH_PRIME64_1; - acc.high64 = 0; - /* - * We set as `i` as offset + 32. We do this so that unchanged - * `len` can be used as upper bound. This reaches a sweet spot - * where both x86 and aarch64 get simple agen and good codegen - * for the loop. - */ - for (i = 32; i < 160; i += 32) { - acc = XXH128_mix32B(acc, - input + i - 32, - input + i - 16, - secret + i - 32, - seed); - } - acc.low64 = XXH3_avalanche(acc.low64); - acc.high64 = XXH3_avalanche(acc.high64); - /* - * NB: `i <= len` will duplicate the last 32-bytes if - * len % 32 was zero. This is an unfortunate necessity to keep - * the hash result stable. - */ - for (i=160; i <= len; i += 32) { - acc = XXH128_mix32B(acc, - input + i - 32, - input + i - 16, - secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, - seed); - } - /* last bytes */ - acc = XXH128_mix32B(acc, - input + len - 16, - input + len - 32, - secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, - (XXH64_hash_t)0 - seed); - - { XXH128_hash_t h128; - h128.low64 = acc.low64 + acc.high64; - h128.high64 = (acc.low64 * XXH_PRIME64_1) - + (acc.high64 * XXH_PRIME64_4) - + ((len - seed) * XXH_PRIME64_2); - h128.low64 = XXH3_avalanche(h128.low64); - h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); - return h128; } + + acc = + XXH128_mix32B(acc, input + 32, input + len - 48, secret + 64, seed); + + } + + acc = XXH128_mix32B(acc, input + 16, input + len - 32, secret + 32, seed); + + } + + acc = XXH128_mix32B(acc, input, input + len - 16, secret, seed); + #endif + { + + XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + + } + } -XXH_FORCE_INLINE XXH128_hash_t -XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, - const xxh_u8* XXH_RESTRICT secret, size_t secretSize, - XXH3_f_accumulate f_acc, - XXH3_f_scrambleAcc f_scramble) -{ - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; - - XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); - - /* converge into final hash */ - XXH_STATIC_ASSERT(sizeof(acc) == 64); - XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); - { XXH128_hash_t h128; - h128.low64 = XXH3_mergeAccs(acc, - secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)len * XXH_PRIME64_1); - h128.high64 = XXH3_mergeAccs(acc, - secret + secretSize - - sizeof(acc) - XXH_SECRET_MERGEACCS_START, - ~((xxh_u64)len * XXH_PRIME64_2)); - return h128; +XXH_NO_INLINE XXH_PUREF XXH128_hash_t XXH3_len_129to240_128b( + const xxh_u8 *XXH_RESTRICT input, size_t len, + const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { + + XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + + acc = XXH128_mix32B(acc, input + i - 32, input + i - 16, secret + i - 32, + seed); + + } + + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i = 160; i <= len; i += 32) { + + acc = XXH128_mix32B(acc, input + i - 32, input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, seed); + + } + + /* last bytes */ + acc = XXH128_mix32B( + acc, input + len - 16, input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { + + XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + + } + +} + +XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_internal( + const void *XXH_RESTRICT input, size_t len, + const xxh_u8 *XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { + + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8 *)input, len, secret, + secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { + + XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs( + acc, secret + secretSize - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + + } + } /* * It's important for performance that XXH3_hashLong() is not inlined. */ -XXH_NO_INLINE XXH_PUREF XXH128_hash_t -XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, - XXH64_hash_t seed64, - const void* XXH_RESTRICT secret, size_t secretLen) -{ - (void)seed64; (void)secret; (void)secretLen; - return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), - XXH3_accumulate, XXH3_scrambleAcc); +XXH_NO_INLINE XXH_PUREF XXH128_hash_t XXH3_hashLong_128b_default( + const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, + const void *XXH_RESTRICT secret, size_t secretLen) { + + (void)seed64; + (void)secret; + (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, + sizeof(XXH3_kSecret), XXH3_accumulate, + XXH3_scrambleAcc); + } /* * It's important for performance to pass @p secretLen (when it's static) * to the compiler, so that it can properly optimize the vectorized loop. * - * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE - * breaks -Og, this is XXH_NO_INLINE. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and + * FORCE_INLINE breaks -Og, this is XXH_NO_INLINE. */ -XXH3_WITH_SECRET_INLINE XXH128_hash_t -XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, - XXH64_hash_t seed64, - const void* XXH_RESTRICT secret, size_t secretLen) -{ - (void)seed64; - return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, - XXH3_accumulate, XXH3_scrambleAcc); +XXH3_WITH_SECRET_INLINE XXH128_hash_t XXH3_hashLong_128b_withSecret( + const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, + const void *XXH_RESTRICT secret, size_t secretLen) { + + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8 *)secret, + secretLen, XXH3_accumulate, + XXH3_scrambleAcc); + } -XXH_FORCE_INLINE XXH128_hash_t -XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, - XXH64_hash_t seed64, - XXH3_f_accumulate f_acc, - XXH3_f_scrambleAcc f_scramble, - XXH3_f_initCustomSecret f_initSec) -{ - if (seed64 == 0) - return XXH3_hashLong_128b_internal(input, len, - XXH3_kSecret, sizeof(XXH3_kSecret), - f_acc, f_scramble); - { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; - f_initSec(secret, seed64); - return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), - f_acc, f_scramble); - } +XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_withSeed_internal( + const void *XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) { + + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, + sizeof(XXH3_kSecret), f_acc, f_scramble); + { + + XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8 *)secret, + sizeof(secret), f_acc, f_scramble); + + } + } /* * It's important for performance that XXH3_hashLong is not inlined. */ XXH_NO_INLINE XXH128_hash_t -XXH3_hashLong_128b_withSeed(const void* input, size_t len, - XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) -{ - (void)secret; (void)secretLen; - return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, - XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +XXH3_hashLong_128b_withSeed(const void *input, size_t len, XXH64_hash_t seed64, + const void *XXH_RESTRICT secret, size_t secretLen) { + + (void)secret; + (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, + XXH3_initCustomSecret); + } -typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, - XXH64_hash_t, const void* XXH_RESTRICT, size_t); +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void *XXH_RESTRICT, size_t, + XXH64_hash_t, + const void *XXH_RESTRICT, size_t); XXH_FORCE_INLINE XXH128_hash_t -XXH3_128bits_internal(const void* input, size_t len, - XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, - XXH3_hashLong128_f f_hl128) -{ - XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); - /* - * If an action is to be taken if `secret` conditions are not respected, - * it should be done here. - * For now, it's a contract pre-condition. - * Adding a check and a branch here would cost performance at every hash. - */ - if (len <= 16) - return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); - if (len <= 128) - return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); - return f_hl128(input, len, seed64, secret, secretLen); -} +XXH3_128bits_internal(const void *input, size_t len, XXH64_hash_t seed64, + const void *XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) { + + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8 *)input, len, + (const xxh_u8 *)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8 *)input, len, + (const xxh_u8 *)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8 *)input, len, + (const xxh_u8 *)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} /* === Public XXH128 API === */ /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) -{ - return XXH3_128bits_internal(input, len, 0, - XXH3_kSecret, sizeof(XXH3_kSecret), - XXH3_hashLong_128b_default); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void *input, + size_t len) { + + return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, + sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); + } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t -XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) -{ - return XXH3_128bits_internal(input, len, 0, - (const xxh_u8*)secret, secretSize, - XXH3_hashLong_128b_withSecret); +XXH3_128bits_withSecret(XXH_NOESCAPE const void *input, size_t len, + XXH_NOESCAPE const void *secret, size_t secretSize) { + + return XXH3_128bits_internal(input, len, 0, (const xxh_u8 *)secret, + secretSize, XXH3_hashLong_128b_withSecret); + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH128_hash_t -XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) -{ - return XXH3_128bits_internal(input, len, seed, - XXH3_kSecret, sizeof(XXH3_kSecret), - XXH3_hashLong_128b_withSeed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed( + XXH_NOESCAPE const void *input, size_t len, XXH64_hash_t seed) { + + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, + sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH128_hash_t -XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) -{ - if (len <= XXH3_MIDSIZE_MAX) - return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); - return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecretandSeed( + XXH_NOESCAPE const void *input, size_t len, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed) { + + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, + sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH128_hash_t -XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) -{ - return XXH3_128bits_withSeed(input, len, seed); -} +XXH_PUBLIC_API XXH128_hash_t XXH128(XXH_NOESCAPE const void *input, size_t len, + XXH64_hash_t seed) { + return XXH3_128bits_withSeed(input, len, seed); + +} -/* === XXH3 128-bit streaming === */ -#ifndef XXH_NO_STREAM + /* === XXH3 128-bit streaming === */ + #ifndef XXH_NO_STREAM /* - * All initialization and update functions are identical to 64-bit streaming variant. - * The only difference is the finalization routine. + * All initialization and update functions are identical to 64-bit streaming + * variant. The only difference is the finalization routine. */ /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) -{ - return XXH3_64bits_reset(statePtr); +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t *statePtr) { + + return XXH3_64bits_reset(statePtr); + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) -{ - return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize) { + + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) -{ - return XXH3_64bits_reset_withSeed(statePtr, seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH64_hash_t seed) { + + return XXH3_64bits_reset_withSeed(statePtr, seed); + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) -{ - return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecretandSeed( + XXH_NOESCAPE XXH3_state_t *statePtr, XXH_NOESCAPE const void *secret, + size_t secretSize, XXH64_hash_t seed) { + + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, + seed); + } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode -XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) -{ - return XXH3_64bits_update(state, input, len); +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t *state, + XXH_NOESCAPE const void *input, size_t len) { + + return XXH3_64bits_update(state, input, len); + } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) -{ - const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; - if (state->totalLen > XXH3_MIDSIZE_MAX) { - XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; - XXH3_digest_long(acc, state, secret); - XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); - { XXH128_hash_t h128; - h128.low64 = XXH3_mergeAccs(acc, - secret + XXH_SECRET_MERGEACCS_START, - (xxh_u64)state->totalLen * XXH_PRIME64_1); - h128.high64 = XXH3_mergeAccs(acc, - secret + state->secretLimit + XXH_STRIPE_LEN - - sizeof(acc) - XXH_SECRET_MERGEACCS_START, - ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); - return h128; - } +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_digest(XXH_NOESCAPE const XXH3_state_t *state) { + + const unsigned char *const secret = + (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= + sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { + + XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = + XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN - + sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } - /* len <= XXH3_MIDSIZE_MAX : short code */ - if (state->seed) - return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); - return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), - secret, state->secretLimit + XXH_STRIPE_LEN); + + } + + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, + state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); + } -#endif /* !XXH_NO_STREAM */ -/* 128-bit utility functions */ -#include /* memcmp, memcpy */ + #endif /* !XXH_NO_STREAM */ + /* 128-bit utility functions */ + + #include /* memcmp, memcpy */ /* return : 1 is equal, 0 if different */ /*! @ingroup XXH3_family */ -XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) -{ - /* note : XXH128_hash_t is compact, it has no padding byte */ - return !(memcmp(&h1, &h2, sizeof(h1))); +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { + + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); + } /* This prototype is compatible with stdlib's qsort(). @@ -6920,129 +7872,156 @@ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) * <0 if *h128_1 < *h128_2 * =0 if *h128_1 == *h128_2 */ /*! @ingroup XXH3_family */ -XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) -{ - XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; - XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; - int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); - /* note : bets that, in most cases, hash values are different */ - if (hcmp) return hcmp; - return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); -} +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void *h128_1, + XXH_NOESCAPE const void *h128_2) { + + XXH128_hash_t const h1 = *(const XXH128_hash_t *)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t *)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} /*====== Canonical representation ======*/ /*! @ingroup XXH3_family */ -XXH_PUBLIC_API void -XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) { - hash.high64 = XXH_swap64(hash.high64); - hash.low64 = XXH_swap64(hash.low64); - } - XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); - XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +XXH_PUBLIC_API void XXH128_canonicalFromHash( + XXH_NOESCAPE XXH128_canonical_t *dst, XXH128_hash_t hash) { + + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + + } + + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char *)dst + sizeof(hash.high64), &hash.low64, + sizeof(hash.low64)); + } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t -XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) -{ - XXH128_hash_t h; - h.high64 = XXH_readBE64(src); - h.low64 = XXH_readBE64(src->digest + 8); - return h; +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t *src) { + + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; + } + /* ========================================== + * Secret generators + * ========================================== + */ + #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) +XXH_FORCE_INLINE void XXH3_combine16(void *dst, XXH128_hash_t h128) { -/* ========================================== - * Secret generators - * ========================================== - */ -#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + XXH_writeLE64(dst, XXH_readLE64(dst) ^ h128.low64); + XXH_writeLE64((char *)dst + 8, XXH_readLE64((char *)dst + 8) ^ h128.high64); -XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) -{ - XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); - XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); } /*! @ingroup XXH3_family */ -XXH_PUBLIC_API XXH_errorcode -XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) -{ -#if (XXH_DEBUGLEVEL >= 1) - XXH_ASSERT(secretBuffer != NULL); - XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); -#else - /* production mode, assert() are disabled */ - if (secretBuffer == NULL) return XXH_ERROR; - if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; -#endif +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret( + XXH_NOESCAPE void *secretBuffer, size_t secretSize, + XXH_NOESCAPE const void *customSeed, size_t customSeedSize) { + + #if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + #else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + #endif + + if (customSeedSize == 0) { + + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + + } + + #if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); + #else + if (customSeed == NULL) return XXH_ERROR; + #endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { + + size_t pos = 0; + while (pos < secretSize) { + + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char *)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; - if (customSeedSize == 0) { - customSeed = XXH3_kSecret; - customSeedSize = XXH_SECRET_DEFAULT_SIZE; } -#if (XXH_DEBUGLEVEL >= 1) - XXH_ASSERT(customSeed != NULL); -#else - if (customSeed == NULL) return XXH_ERROR; -#endif - /* Fill secretBuffer with a copy of customSeed - repeat as needed */ - { size_t pos = 0; - while (pos < secretSize) { - size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); - memcpy((char*)secretBuffer + pos, customSeed, toCopy); - pos += toCopy; - } } - - { size_t const nbSeg16 = secretSize / 16; - size_t n; - XXH128_canonical_t scrambler; - XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); - for (n=0; ninit_seed = init_seed; diff --git a/utils/bench/hash.c b/utils/bench/hash.c index 013a5321..d4be0ab4 100644 --- a/utils/bench/hash.c +++ b/utils/bench/hash.c @@ -13,30 +13,41 @@ #undef XXH_INLINE_ALL int main() { - char *data = malloc(4097); + + char *data = malloc(4097); struct timespec start, end; - long long duration; - int i; - uint64_t res; + long long duration; + int i; + uint64_t res; clock_gettime(CLOCK_MONOTONIC, &start); for (i = 0; i < 100000000; ++i) { - res = XXH3_64bits(data, 4097); - memcpy(data + 16, (char*)&res, 8); + + res = XXH3_64bits(data, 4097); + memcpy(data + 16, (char *)&res, 8); + } + clock_gettime(CLOCK_MONOTONIC, &end); - duration = (end.tv_sec - start.tv_sec) * 1000000000LL + (end.tv_nsec - start.tv_nsec); + duration = (end.tv_sec - start.tv_sec) * 1000000000LL + + (end.tv_nsec - start.tv_nsec); printf("xxh3 duration: %lld ns\n", duration); memset(data, 0, 4097); clock_gettime(CLOCK_MONOTONIC, &start); for (i = 0; i < 100000000; ++i) { - res = t1ha0_ia32aes(data, 4097); - memcpy(data + 16, (char*)&res, 8); + + res = t1ha0_ia32aes(data, 4097); + memcpy(data + 16, (char *)&res, 8); + } + clock_gettime(CLOCK_MONOTONIC, &end); - duration = (end.tv_sec - start.tv_sec) * 1000000000LL + (end.tv_nsec - start.tv_nsec); + duration = (end.tv_sec - start.tv_sec) * 1000000000LL + + (end.tv_nsec - start.tv_nsec); printf("t1ha0_ia32aes duration: %lld ns\n", duration); return 0; + } + -- cgit 1.4.1 From 8fcd404352a0bcf1084c518e779fe042c61d2b9e Mon Sep 17 00:00:00 2001 From: Joachim Hyrathon Date: Tue, 27 Feb 2024 15:34:42 +0800 Subject: Update GNUmakefile The linker flags lacks a -ldl so the dlopen series of func symbols can't be found --- utils/afl_network_proxy/GNUmakefile | 1 + 1 file changed, 1 insertion(+) (limited to 'utils') diff --git a/utils/afl_network_proxy/GNUmakefile b/utils/afl_network_proxy/GNUmakefile index 7c8c22ff..47d9a7d3 100644 --- a/utils/afl_network_proxy/GNUmakefile +++ b/utils/afl_network_proxy/GNUmakefile @@ -10,6 +10,7 @@ PROGRAMS = afl-network-client afl-network-server HASH=\# CFLAGS += -Wno-pointer-sign +LDFLAGS += -ldl ifdef STATIC CFLAGS += -static -- cgit 1.4.1