diff options
| author | Martin Storsjo <martin@martin.st> | 2020-09-02 12:55:33 +0300 |
|---|---|---|
| committer | Martin Storsjo <martin@martin.st> | 2020-09-02 12:56:41 +0300 |
| commit | 3a831a5fbc990c83e9b5b804a082bb158364e793 (patch) | |
| tree | c628b7dfbd683d01015351c18af1bab5fc8a5329 /libFDK/include/scale.h | |
| parent | 8439b745f65bce3fd55ffc9f9edcc04f5f447c55 (diff) | |
| download | fdk-aac-3a831a5fbc990c83e9b5b804a082bb158364e793.tar.gz fdk-aac-3a831a5fbc990c83e9b5b804a082bb158364e793.tar.bz2 fdk-aac-3a831a5fbc990c83e9b5b804a082bb158364e793.zip | |
Avoid undefined shifts in SATURATE_SHIFT
Make sure that the shift amount is less than the size of the shifted
value, otherwise return the saturation max values (for left shift, if
the source values was nonzero) or zero (for right shift, or zero
shifted left).
Fixes: 24376/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_LIBFDK_AAC_fuzzer-6529411206348800
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Diffstat (limited to 'libFDK/include/scale.h')
| -rw-r--r-- | libFDK/include/scale.h | 15 |
1 files changed, 13 insertions, 2 deletions
diff --git a/libFDK/include/scale.h b/libFDK/include/scale.h index 655ccaf..fcfc116 100644 --- a/libFDK/include/scale.h +++ b/libFDK/include/scale.h @@ -241,20 +241,31 @@ inline void scaleValueInPlace(FIXP_DBL *value, /*!< Value */ #ifndef SATURATE_RIGHT_SHIFT #define SATURATE_RIGHT_SHIFT(src, scale, dBits) \ + (((scale) >= 8*sizeof(LONG)) ? (LONG)0 : \ ((((LONG)(src) >> (scale)) > (LONG)(((1U) << ((dBits)-1)) - 1)) \ ? (LONG)(((1U) << ((dBits)-1)) - 1) \ : (((LONG)(src) >> (scale)) < ~((LONG)(((1U) << ((dBits)-1)) - 1))) \ ? ~((LONG)(((1U) << ((dBits)-1)) - 1)) \ - : ((LONG)(src) >> (scale))) + : ((LONG)(src) >> (scale)))) +#endif + +#ifndef SATURATE_LEFT_MAX +#define SATURATE_LEFT_MAX(src, dBits) \ + (((LONG)(src) > 0) \ + ? (LONG)(((1U) << ((dBits)-1)) - 1) \ + : ((LONG)(src) < 0) \ + ? ~((LONG)(((1U) << ((dBits)-1)) - 1)) \ + : (LONG)0) #endif #ifndef SATURATE_LEFT_SHIFT #define SATURATE_LEFT_SHIFT(src, scale, dBits) \ + (((scale) >= 8*sizeof(LONG)) ? SATURATE_LEFT_MAX(src, dBits) : \ (((LONG)(src) > ((LONG)(((1U) << ((dBits)-1)) - 1) >> (scale))) \ ? (LONG)(((1U) << ((dBits)-1)) - 1) \ : ((LONG)(src) < ~((LONG)(((1U) << ((dBits)-1)) - 1) >> (scale))) \ ? ~((LONG)(((1U) << ((dBits)-1)) - 1)) \ - : ((LONG)(src) << (scale))) + : ((LONG)(src) << (scale)))) #endif #ifndef SATURATE_SHIFT |