summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Storsjo <martin@martin.st>2012-08-10 10:57:24 +0300
committerMartin Storsjo <martin@martin.st>2012-08-10 11:05:56 +0300
commit65745694963cc601a2403d071311261f200d6cb9 (patch)
tree43d650bf77368c1769881c4bcb65fdf04e51efab
parentd2854e2ac7e7e3cbe4b3c67a2d7823a6153c2488 (diff)
downloadODR-AudioEnc-65745694963cc601a2403d071311261f200d6cb9.tar.gz
ODR-AudioEnc-65745694963cc601a2403d071311261f200d6cb9.tar.bz2
ODR-AudioEnc-65745694963cc601a2403d071311261f200d6cb9.zip
Don't try to shift more bits than the variable length
Shifting by more than (or equal to) the variable length is undefined in C. This caused the quantized spectrum values to go out of range, triggering asserts later. Change-Id: If81b6c8caa7b9c75941ad9d280b686d2069c968c
-rw-r--r--libAACenc/src/quantize.cpp10
1 files changed, 8 insertions, 2 deletions
diff --git a/libAACenc/src/quantize.cpp b/libAACenc/src/quantize.cpp
index 9694901..dc85a6d 100644
--- a/libAACenc/src/quantize.cpp
+++ b/libAACenc/src/quantize.cpp
@@ -127,7 +127,10 @@ static void FDKaacEnc_quantizeLines(INT gain,
accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
totalShift = (16-4)-(3*(totalShift>>2));
FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
- accu>>=totalShift;
+ if (totalShift < 32)
+ accu>>=totalShift;
+ else
+ accu = 0;
quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16)));
}
else if(accu > FL2FXCONST_DBL(0.0f))
@@ -140,7 +143,10 @@ static void FDKaacEnc_quantizeLines(INT gain,
accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
totalShift = (16-4)-(3*(totalShift>>2));
FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
- accu>>=totalShift;
+ if (totalShift < 32)
+ accu>>=totalShift;
+ else
+ accu = 0;
quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16));
}
else