From 65745694963cc601a2403d071311261f200d6cb9 Mon Sep 17 00:00:00 2001 From: Martin Storsjo Date: Fri, 10 Aug 2012 10:57:24 +0300 Subject: Don't try to shift more bits than the variable length Shifting by more than (or equal to) the variable length is undefined in C. This caused the quantized spectrum values to go out of range, triggering asserts later. Change-Id: If81b6c8caa7b9c75941ad9d280b686d2069c968c --- libAACenc/src/quantize.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libAACenc/src/quantize.cpp b/libAACenc/src/quantize.cpp index 9694901..dc85a6d 100644 --- a/libAACenc/src/quantize.cpp +++ b/libAACenc/src/quantize.cpp @@ -127,7 +127,10 @@ static void FDKaacEnc_quantizeLines(INT gain, accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); totalShift = (16-4)-(3*(totalShift>>2)); FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ - accu>>=totalShift; + if (totalShift < 32) + accu>>=totalShift; + else + accu = 0; quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16))); } else if(accu > FL2FXCONST_DBL(0.0f)) @@ -140,7 +143,10 @@ static void FDKaacEnc_quantizeLines(INT gain, accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); totalShift = (16-4)-(3*(totalShift>>2)); FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ - accu>>=totalShift; + if (totalShift < 32) + accu>>=totalShift; + else + accu = 0; quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16)); } else -- cgit v1.2.3