diff options
| author | Martin Storsjo <martin@martin.st> | 2012-08-10 10:57:24 +0300 |
|---|---|---|
| committer | Martin Storsjo <martin@martin.st> | 2012-08-10 11:05:56 +0300 |
| commit | 65745694963cc601a2403d071311261f200d6cb9 (patch) | |
| tree | 43d650bf77368c1769881c4bcb65fdf04e51efab | |
| parent | d2854e2ac7e7e3cbe4b3c67a2d7823a6153c2488 (diff) | |
| download | fdk-aac-65745694963cc601a2403d071311261f200d6cb9.tar.gz fdk-aac-65745694963cc601a2403d071311261f200d6cb9.tar.bz2 fdk-aac-65745694963cc601a2403d071311261f200d6cb9.zip | |
Don't try to shift more bits than the variable length
Shifting by more than (or equal to) the variable length is
undefined in C.
This caused the quantized spectrum values to go out of range,
triggering asserts later.
Change-Id: If81b6c8caa7b9c75941ad9d280b686d2069c968c
| -rw-r--r-- | libAACenc/src/quantize.cpp | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/libAACenc/src/quantize.cpp b/libAACenc/src/quantize.cpp index 9694901..dc85a6d 100644 --- a/libAACenc/src/quantize.cpp +++ b/libAACenc/src/quantize.cpp @@ -127,7 +127,10 @@ static void FDKaacEnc_quantizeLines(INT gain, accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); totalShift = (16-4)-(3*(totalShift>>2)); FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ - accu>>=totalShift; + if (totalShift < 32) + accu>>=totalShift; + else + accu = 0; quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16))); } else if(accu > FL2FXCONST_DBL(0.0f)) @@ -140,7 +143,10 @@ static void FDKaacEnc_quantizeLines(INT gain, accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]); totalShift = (16-4)-(3*(totalShift>>2)); FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */ - accu>>=totalShift; + if (totalShift < 32) + accu>>=totalShift; + else + accu = 0; quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16)); } else |