From 8f9f42c1041e877c602e0acd43361d7068c3ce1c Mon Sep 17 00:00:00 2001 From: Martin Storsjo Date: Sun, 17 Feb 2013 02:14:35 +0200 Subject: Don't try to shift more bits than the variable length Shifting by more than (or equal to) the variable length is undefined in C. This caused the quantized spectrum values to remain unshifted, causing loud beeps in some samples. The exact same change was originally applied in 657456949 (for AOSP) and in 24021f190 (in the separate fdk-aac repo), but was overwritten and silently reverted by the new upstream code drop for Android 4.2 in 381d69840a. After the code drop, I chose not to reapply this change since the crashes that it fixed had been fixed in other ways upstream and I was unable to reproduce them after the new code drop. --- libAACenc/src/quantize.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'libAACenc/src/quantize.cpp') diff --git a/libAACenc/src/quantize.cpp b/libAACenc/src/quantize.cpp index a1698a8..1f7402b 100644 --- a/libAACenc/src/quantize.cpp +++ b/libAACenc/src/quantize.cpp @@ -127,7 +127,10 @@ static void FDKaacEnc_quantizeLines(INT gain, accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); totalShift = (16-4)-(3*(totalShift>>2)); FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ - accu>>=totalShift; + if (totalShift < 32) + accu>>=totalShift; + else + accu = 0; quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16))); } else if(accu > FL2FXCONST_DBL(0.0f)) @@ -140,7 +143,10 @@ static void FDKaacEnc_quantizeLines(INT gain, accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); totalShift = (16-4)-(3*(totalShift>>2)); FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ - accu>>=totalShift; + if (totalShift < 32) + accu>>=totalShift; + else + accu = 0; quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16)); } else -- cgit v1.2.3