From b5e0fff6b9ee1260fc3753af0e441606cfe853ff Mon Sep 17 00:00:00 2001 From: Kuan-Wei Chiu Date: Fri, 14 Jun 2024 09:44:36 +0800 Subject: Fix signed integer overflow in radamsa When UBSan was enabled and 'make test' was executed, the following runtime error was observed: libradamsa.c:26055:29: runtime error: left shift of 16777215 by 8 places cannot be represented in type 'int' This issue was caused by a left shift operation on a signed integer. The fix involved changing the integer literal to an unsigned integer by adding a 'U' suffix, ensuring the left shift operation produces the correct value without overflow. Fixes: 8178f4df ("remove radamsa, add radamsa custom mutator") --- custom_mutators/radamsa/libradamsa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'custom_mutators') diff --git a/custom_mutators/radamsa/libradamsa.c b/custom_mutators/radamsa/libradamsa.c index e6838752..1dcf91d8 100644 --- a/custom_mutators/radamsa/libradamsa.c +++ b/custom_mutators/radamsa/libradamsa.c @@ -3707,7 +3707,7 @@ typedef intptr_t wdiff; 1024 * 1024 * 8 /* static malloc'd heap size if used as a library */ #define FBITS 24 /* bits in fixnum, on the way to 24 and beyond */ #define FMAX \ - ((1 << FBITS) - 1) /* maximum fixnum (and most negative fixnum) \ + ((1U << FBITS) - 1) /* maximum fixnum (and most negative fixnum) \ */ #define MAXOBJ 0xffff /* max words in tuple including header */ #define MAXPAYL \ -- cgit 1.4.1