From ed247dfa54234c1115c35a05327955b29b161e8a Mon Sep 17 00:00:00 2001 From: Jean-Michel Trivi Date: Thu, 8 Aug 2013 17:38:33 -0700 Subject: Encoder quantizer fix AAC Encoder: Saturate quantizer shift value to prevent undefined behaviour. In very rare cases the shift value 'totalShift = (16-4)-(3*(totalShift>>2))' can be greater than accu data width. If you apply a shift with more then 31 bit the result depends on the architecture and is not defined in C. For certain platforms zeros are shifted in. That would be our desired behaviour. On other platforms the shift will be applied as modulo. For example >>34 would be applied as >>2. To prevent this discrepancy the shift value is limited/saturated to DFRACT_BITS-1. 'accu >>= fixMin(totalShift,DFRACT_BITS-1)'. Bug 9428126 Change-Id: I27177654c4dc22cf899bc35dad9cdd040dccb02d --- libAACenc/src/quantize.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libAACenc/src/quantize.cpp') diff --git a/libAACenc/src/quantize.cpp b/libAACenc/src/quantize.cpp index da6f85e..5380e35 100644 --- a/libAACenc/src/quantize.cpp +++ b/libAACenc/src/quantize.cpp @@ -127,7 +127,7 @@ static void FDKaacEnc_quantizeLines(INT gain, accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); totalShift = (16-4)-(3*(totalShift>>2)); FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ - accu>>=totalShift; + accu >>= fixMin(totalShift,DFRACT_BITS-1); quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16))); } else if(accu > FL2FXCONST_DBL(0.0f)) @@ -140,7 +140,7 @@ static void FDKaacEnc_quantizeLines(INT gain, accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); totalShift = (16-4)-(3*(totalShift>>2)); FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ - accu>>=totalShift; + accu >>= fixMin(totalShift,DFRACT_BITS-1); quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16)); } else -- cgit v1.2.3