From 3a831a5fbc990c83e9b5b804a082bb158364e793 Mon Sep 17 00:00:00 2001 From: Martin Storsjo Date: Wed, 2 Sep 2020 12:55:33 +0300 Subject: Avoid undefined shifts in SATURATE_SHIFT Make sure that the shift amount is less than the size of the shifted value, otherwise return the saturation max values (for left shift, if the source values was nonzero) or zero (for right shift, or zero shifted left). Fixes: 24376/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_LIBFDK_AAC_fuzzer-6529411206348800 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg --- libFDK/include/scale.h | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'libFDK/include') diff --git a/libFDK/include/scale.h b/libFDK/include/scale.h index 655ccaf..fcfc116 100644 --- a/libFDK/include/scale.h +++ b/libFDK/include/scale.h @@ -241,20 +241,31 @@ inline void scaleValueInPlace(FIXP_DBL *value, /*!< Value */ #ifndef SATURATE_RIGHT_SHIFT #define SATURATE_RIGHT_SHIFT(src, scale, dBits) \ + (((scale) >= 8*sizeof(LONG)) ? (LONG)0 : \ ((((LONG)(src) >> (scale)) > (LONG)(((1U) << ((dBits)-1)) - 1)) \ ? (LONG)(((1U) << ((dBits)-1)) - 1) \ : (((LONG)(src) >> (scale)) < ~((LONG)(((1U) << ((dBits)-1)) - 1))) \ ? ~((LONG)(((1U) << ((dBits)-1)) - 1)) \ - : ((LONG)(src) >> (scale))) + : ((LONG)(src) >> (scale)))) +#endif + +#ifndef SATURATE_LEFT_MAX +#define SATURATE_LEFT_MAX(src, dBits) \ + (((LONG)(src) > 0) \ + ? (LONG)(((1U) << ((dBits)-1)) - 1) \ + : ((LONG)(src) < 0) \ + ? ~((LONG)(((1U) << ((dBits)-1)) - 1)) \ + : (LONG)0) #endif #ifndef SATURATE_LEFT_SHIFT #define SATURATE_LEFT_SHIFT(src, scale, dBits) \ + (((scale) >= 8*sizeof(LONG)) ? SATURATE_LEFT_MAX(src, dBits) : \ (((LONG)(src) > ((LONG)(((1U) << ((dBits)-1)) - 1) >> (scale))) \ ? (LONG)(((1U) << ((dBits)-1)) - 1) \ : ((LONG)(src) < ~((LONG)(((1U) << ((dBits)-1)) - 1) >> (scale))) \ ? ~((LONG)(((1U) << ((dBits)-1)) - 1)) \ - : ((LONG)(src) << (scale))) + : ((LONG)(src) << (scale)))) #endif #ifndef SATURATE_SHIFT -- cgit v1.2.3