error handling for custom mutators

2 files changed, 10 insertions, 9 deletions

diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h
index 79878cb6..a265c1a3 100644
--- a/include/afl-fuzz.h
+++ b/include/afl-fuzz.h
@@ -700,9 +700,10 @@ struct custom_mutator {
    * @param data pointer returned in afl_custom_init for this fuzz case
    * @param buf Buffer containing the test case
    * @param buf_size Size of the test case
-   * @return The amount of possible iteration steps to trim the input
+   * @return The amount of possible iteration steps to trim the input.
+   *        Negative on error.
    */
-  u32 (*afl_custom_init_trim)(void *data, u8 *buf, size_t buf_size);
+  s32 (*afl_custom_init_trim)(void *data, u8 *buf, size_t buf_size);
 
   /**
    * This method is called for each trimming operation. It doesn't have any
@@ -733,9 +734,9 @@ struct custom_mutator {
    * @param data pointer returned in afl_custom_init for this fuzz case
    * @param success Indicates if the last trim operation was successful.
    * @return The next trim iteration index (from 0 to the maximum amount of
-   *     steps returned in init_trim)
+   *     steps returned in init_trim). Negative on error.
    */
-  u32 (*afl_custom_post_trim)(void *data, u8 success);
+  s32 (*afl_custom_post_trim)(void *data, u8 success);
 
   /**
    * Perform a single custom mutation on a given input.
@@ -818,8 +819,8 @@ u8   trim_case_custom(afl_state_t *, struct queue_entry *q, u8 *in_buf);
 void finalize_py_module(void *);
 
 size_t pre_save_py(void *, u8 *, size_t, u8 **);
-u32    init_trim_py(void *, u8 *, size_t);
-u32    post_trim_py(void *, u8);
+s32    init_trim_py(void *, u8 *, size_t);
+s32    post_trim_py(void *, u8);
 size_t trim_py(void *, u8 **);
 size_t havoc_mutation_py(void *, u8 *, size_t, u8 **, size_t);
 u8     havoc_mutation_probability_py(void *);
diff --git a/include/alloc-inl.h b/include/alloc-inl.h
index b8c83db4..91564932 100644
--- a/include/alloc-inl.h
+++ b/include/alloc-inl.h
@@ -767,10 +767,10 @@ static inline void TRK_ck_free(void *ptr, const char *file, const char *func,
 
 #endif                                          /* _WANT_ORIGINAL_AFL_ALLOC */
 
-/* This function calculates the lowest power of 2 greater or equal its argument.
+/* This function calculates the next power of 2 greater or equal its argument.
  @return The rounded up power of 2 (if no overflow) or 0 on overflow.
 */
-static inline size_t powerOf2Ceil(size_t in) {
+static inline size_t next_pow2(size_t in) {
   if (in == 0 || in > (size_t)-1) return 0; /* avoid undefined behaviour under-/overflow */
   size_t out = in - 1;
   out |= out >> 1;
@@ -801,7 +801,7 @@ static inline void *ck_maybe_grow(void **buf, size_t *size,
   if (size_needed < INITIAL_GROWTH_SIZE) size_needed = INITIAL_GROWTH_SIZE;
 
   /* grow exponentially */
-  size_t next_size = powerOf2Ceil(size_needed);
+  size_t next_size = next_pow2(size_needed);
 
   /* handle overflow */
   if (!next_size) {