Bug fixes in AAC decoder, AAC encoder, FDK library

Latest code drop from Fraunhofer:

* AAC-Decoder

   - Expanded AAC-LD/ELD decoder TNS max band tables to avoid wrong data
     access for sampling rates <22kHz and >48kHz.
     Modified file(s):
        libAACdec\src\aacdec_tns.cpp
        libAACdec\src\aac_rom.h
        libAACdec\src\aac_rom.cpp

  - Fixed ELD synthesis QMF filterbank scaling for downsampled SBR.
    Modified file(s):
        libFDK\src\qmf.cpp

* AAC-Encoder

  - Fixed bit rate limiting for lower limit as introduced in the Delivery 2012-05-11.
    Modified file(s):
        libAACenc\include\aacenc_lib.h
        libAACenc\src\aacenc.h
        libAACenc\src\aacenc.cpp
        libSBRenc\src\sbr_rom.h
        libSBRenc\src\sbr_rom.cpp

  - Allow negative prediction gain as meaning that there is no coding gain.
    Make use of scaleValueSaturate in gauss window calculation.
    Modified file(s):
        libAACenc\src\aacenc_tns.cpp

  - Added energy saturation to prevent overflow in short blocks.
    Modified file(s):
        libAACenc\src\grp_data.cpp

  - Perform scalefactor adaption in case quantized lines are out of valid range.
    Modified file(s):
        libAACenc\src\quantize.cpp

  - Interrupt quantization loop when no spectral data is available.
    Adjustments in bitreservoir adaption.
    Modified file(s):
        libAACenc\src\qc_main.cpp

  - Indroduced dynamic scaling in none missing harmonic energy lowering compensation.
    Modified file(s):
        libSBRenc\src\env_est.cpp

* FDK-Library

  - Fixed saturation for negative values in scaleValueSaturate.
    Modified file(s):
        libFDK\include\scale.h

Change-Id: If830ea65caef6b5554281e4b7a77a8b2e08825ce

8 files changed, 79 insertions, 151 deletions

diff --git a/libAACenc/include/aacenc_lib.h b/libAACenc/include/aacenc_lib.h
index 4635119..862dcb5 100644
--- a/libAACenc/include/aacenc_lib.h
+++ b/libAACenc/include/aacenc_lib.h
@@ -424,102 +424,11 @@ For HE-AAC and HE-AAC v2 the lowest possible audio input sampling frequency is 1
 AAC-LC core encoder operates in dual rate mode at its lowest possible sampling frequency, which is 8 kHz.
 HE-AAC v2 requires stereo input audio data.
 
-The following table lists the supported bitrates for AAC-LC, HE-AAC and HE-AAC v2 encoding depending
-on input sampling frequency ("Hz") and number of input channels ("chan"). The minimum and maximum
-allowed bitrate ("BR Min", "BR Max") is given in bits per second.
-In case the desired combination of bitrate and sampling frequency is not available ("NA") for HE-AAC or
-HE-AAC v2 then the encoder will automatically switch to AAC-LC and give a command line warning.
 Please note that in HE-AAC or HE-AAC v2 mode the encoder supports much higher bitrates than are
 appropriate for HE-AAC or HE-AAC v2. For example, at a bitrate of more than 64 kbit/s for a stereo
 audio signal at 44.1 kHz it usually makes sense to use AAC-LC, which will produce better audio
 quality at that bitrate than HE-AAC or HE-AAC v2.
 
-
-\verbatim
-  Config            AAC-LC                  HE-AAC (SBR)          HE-AACv2 (SBR+PS)
-
- Hz   chan       BR Min   BR Max          BR Min   BR Max          BR Min   BR Max
-
-8000    1        758      48000           NA       NA               NA       NA
-11025   1        1045     66150           NA       NA               NA       NA
-12000   1        1137     72000           NA       NA               NA       NA
-16000   1        1516     96000           8000     48000            NA       NA
-22050   1        2089     132300          8000     64000            NA       NA
-24000   1        2274     144000          8000     64000            NA       NA
-32000   1        3032     192000          8000     64000            NA       NA
-44100   1        4178     264576          8000     64000            NA       NA
-48000   1        4547     288000          12000    64000            NA       NA
-64000   1        6063     384000          24000    160000           NA       NA
-88200   1        8355     529200          24000    160000           NA       NA
-96000   1        9094     576000          24000    160000           NA       NA
------------------------------------------------------------------------------------
-8000    2        1071     96000           NA       NA               NA       NA
-11025   2        1476     132300          NA       NA               NA       NA
-12000   2        1606     144000          NA       NA               NA       NA
-16000   2        2141     192000          16000    96000            8000     48000
-22050   2        2951     264600          16000    128000           8000     64000
-24000   2        3211     288000          16000    128000           8000     64000
-32000   2        4282     384000          16000    128000           8000     64000
-44100   2        5900     529152          16000    128000           8000     64000
-48000   2        6422     576000          16000    128000           12000    64000
-64000   2        8563     768000          32000    256000           24000    160000
-88200   2        11801    1058400         32000    256000           24000    160000
-96000   2        12844    1152000         32000    256000           24000    160000
------------------------------------------------------------------------------------
-8000    3        1383     144000          NA       NA               NA       NA
-11025   3        1906     198450          NA       NA               NA       NA
-12000   3        2075     216000          NA       NA               NA       NA
-16000   3        2766     288000          26667    120000           NA       NA
-22050   3        3812     396900          26667    160000           NA       NA
-24000   3        4149     432000          26667    160000           NA       NA
-32000   3        5532     576000          26667    160000           NA       NA
-44100   3        7623     793728          26667    160000           NA       NA
-48000   3        8297     864000          29996    160000           NA       NA
-64000   3        11063    1152000         59996    400000           NA       NA
-88200   3        15246    1587600         59996    400000           NA       NA
-96000   3        16594    1728000         59996    400000           NA       NA
------------------------------------------------------------------------------------
-8000    4        1696     192000          NA       NA               NA       NA
-11025   4        2337     264600          NA       NA               NA       NA
-12000   4        2543     288000          NA       NA               NA       NA
-16000   4        3391     384000          40000    160000           NA       NA
-22050   4        4673     529200          40000    213330           NA       NA
-24000   4        5086     576000          40000    213330           NA       NA
-32000   4        6782     768000          40000    213330           NA       NA
-44100   4        9345     1058304         40000    213330           NA       NA
-48000   4        10172    1152000         40000    213330           NA       NA
-64000   4        13563    1536000         80000    533330           NA       NA
-88200   4        18691    2116800         80000    533330           NA       NA
-96000   4        20344    2304000         80000    533330           NA       NA
------------------------------------------------------------------------------------
-8000    5        2008     240000          NA       NA               NA       NA
-11025   5        2768     330750          NA       NA               NA       NA
-12000   5        3012     360000          NA       NA               NA       NA
-16000   5        4016     480000          43244    184612           NA       NA
-22050   5        5535     661500          43244    246152           NA       NA
-24000   5        6024     720000          43244    246152           NA       NA
-32000   5        8032     960000          43244    246152           NA       NA
-44100   5        11068    1322880         43244    246152           NA       NA
-48000   5        12047    1440000         46140    246152           NA       NA
-64000   5        16063    1920000         92296    615384           NA       NA
-88200   5        22137    2646000         92296    615384           NA       NA
-96000   5        24094    2880000         92296    615384           NA       NA
------------------------------------------------------------------------------------
-8000    5.1      2321     240000          NA       NA               NA       NA
-11025   5.1      3198     330750          NA       NA               NA       NA
-12000   5.1      3481     360000          NA       NA               NA       NA
-16000   5.1      4641     480000          45715    199990           NA       NA
-22050   5.1      6396     661500          45715    266658           NA       NA
-24000   5.1      6961     720000          45715    266658           NA       NA
-32000   5.1      9282     960000          45715    266658           NA       NA
-44100   5.1      12790    1322880         45715    266658           NA       NA
-48000   5.1      13922    1440000         49982    266658           NA       NA
-64000   5.1      18563    1920000         99982    666658           NA       NA
-88200   5.1      25582    2646000         99982    666658           NA       NA
-96000   5.1      27844    2880000         99982    666658           NA       NA
-
-\endverbatim \n
-
 \section reommendedConfig Recommended Sampling Rate and Bitrate Combinations
 
 The following table provides an overview of recommended encoder configuration parameters
@@ -956,8 +865,8 @@ typedef enum
 
   AACENC_GRANULE_LENGTH           = 0x0105,  /*!< Core encoder (AAC) audio frame length in samples:
                                                   - 1024: Default configuration.
-                                                  - 512: Optional length in LD/ELD configuration.
-                                                  - 480: Default LD/ELD configuration. */
+                                                  - 512: Default LD/ELD configuration.
+                                                  - 480: Optional length in LD/ELD configuration. */
 
   AACENC_CHANNELMODE              = 0x0106,  /*!< Set explicit channel mode. Channel mode must match with number of input channels.
                                                   - 1-6: MPEG channel modes supported, see ::CHANNEL_MODE in FDK_audio.h. */
diff --git a/libAACenc/src/aacenc.cpp b/libAACenc/src/aacenc.cpp
index 85083cd..d3f36aa 100644
--- a/libAACenc/src/aacenc.cpp
+++ b/libAACenc/src/aacenc.cpp
@@ -149,7 +149,7 @@ INT FDKaacEnc_LimitBitrate(
       transportBits = 208;
     }
 
-    bitRate = FDKmax(bitRate, ((((40 * nChannels) + transportBits + frameLength) * (coreSamplingRate)) / frameLength) );
+    bitRate = FDKmax(bitRate, ((((40 * nChannels) + transportBits) * (coreSamplingRate)) / frameLength) );
     FDK_ASSERT(bitRate >= 0);
 
     bitRate = FDKmin(bitRate, ((nChannelsEff * MIN_BUFSIZE_PER_EFF_CHAN)*(coreSamplingRate>>shift)) / (frameLength>>shift)) ;
@@ -280,7 +280,7 @@ void FDKaacEnc_AacInitDefaultConfig(AACENC_CONFIG *config)
     config->useTns          = TNS_ENABLE_MASK;      /* tns enabled completly */
     config->usePns          = 1;                    /* depending on channelBitrate this might be set to 0 later */
     config->useIS           = 1;                    /* Intensity Stereo Configuration */
-    config->framelength     = DEFAULT_FRAMELENGTH;  /* used frame size */
+    config->framelength     = -1;                   /* Framesize not configured */
     config->syntaxFlags     = 0;                    /* default syntax with no specialities */
     config->epConfig        = -1;                   /* no ER syntax -> no additional error protection */
     config->nSubFrames      = 1;                    /* default, no sub frames */
@@ -451,11 +451,8 @@ AAC_ENCODER_ERROR FDKaacEnc_Initialize(HANDLE_AAC_ENC      hAacEnc,
   switch (config->framelength)
   {
     case 1024:
-      if ( config->audioObjectType != AOT_AAC_LC
-        && config->audioObjectType != AOT_SBR
-        && config->audioObjectType != AOT_PS
-        && config->audioObjectType != AOT_ER_AAC_LC
-        && config->audioObjectType != AOT_AAC_SCAL )
+      if ( config->audioObjectType == AOT_ER_AAC_LD
+        || config->audioObjectType == AOT_ER_AAC_ELD )
       {
         return AAC_ENC_INVALID_FRAME_LENGTH;
       }
diff --git a/libAACenc/src/aacenc.h b/libAACenc/src/aacenc.h
index 8242248..942f686 100644
--- a/libAACenc/src/aacenc.h
+++ b/libAACenc/src/aacenc.h
@@ -153,7 +153,6 @@ typedef enum {
 /*-------------------------- defines --------------------------------------*/
 
 #define ANC_DATA_BUFFERSIZE 1024      /* ancBuffer size */
-#define DEFAULT_FRAMELENGTH 1024      /* size of AAC core frame in (new) PCM samples */
 
 #define MAX_TOTAL_EXT_PAYLOADS  (((6) * (1)) + (2+2))
 
diff --git a/libAACenc/src/aacenc_lib.cpp b/libAACenc/src/aacenc_lib.cpp
index cbb0e2a..a245358 100644
--- a/libAACenc/src/aacenc_lib.cpp
+++ b/libAACenc/src/aacenc_lib.cpp
@@ -98,7 +98,7 @@ amm-info@iis.fraunhofer.de
 /* Encoder library info */
 #define AACENCODER_LIB_VL0 3
 #define AACENCODER_LIB_VL1 3
-#define AACENCODER_LIB_VL2 1
+#define AACENCODER_LIB_VL2 2
 #define AACENCODER_LIB_TITLE "AAC Encoder"
 #define AACENCODER_LIB_BUILD_DATE __DATE__
 #define AACENCODER_LIB_BUILD_TIME __TIME__
diff --git a/libAACenc/src/aacenc_tns.cpp b/libAACenc/src/aacenc_tns.cpp
index 933e4e7..00b2bca 100644
--- a/libAACenc/src/aacenc_tns.cpp
+++ b/libAACenc/src/aacenc_tns.cpp
@@ -1067,11 +1067,11 @@ static void FDKaacEnc_CalcGaussWindow(
         const INT timeResolution_e
         )
 {
-  #define PI_SCALE         (2)
-  #define PI_FIX           FL2FXCONST_DBL(3.1416f/(float)(1<<PI_SCALE))
+  #define PI_E           (2)
+  #define PI_M           FL2FXCONST_DBL(3.1416f/(float)(1<<PI_E))
 
-  #define EULER_SCALE      (2)
-  #define EULER_FIX        FL2FXCONST_DBL(2.7183/(float)(1<<EULER_SCALE))
+  #define EULER_E        (2)
+  #define EULER_M        FL2FXCONST_DBL(2.7183/(float)(1<<EULER_E))
 
   #define COEFF_LOOP_SCALE (4)
 
@@ -1083,9 +1083,9 @@ static void FDKaacEnc_CalcGaussWindow(
    *   gaussExp = PI * samplingRate * 0.001f * timeResolution / transformResolution;
    *   gaussExp = -0.5f * gaussExp * gaussExp;
    */
-  gaussExp_m = fMultNorm(timeResolution, fMult(PI_FIX, fDivNorm( (FIXP_DBL)(samplingRate), (FIXP_DBL)(LONG)(transformResolution*1000.f), &e1)), &e2);
+  gaussExp_m = fMultNorm(timeResolution, fMult(PI_M, fDivNorm( (FIXP_DBL)(samplingRate), (FIXP_DBL)(LONG)(transformResolution*1000.f), &e1)), &e2);
   gaussExp_m = -fPow2Div2(gaussExp_m);
-  gaussExp_e = 2*(e1+e2+timeResolution_e+PI_SCALE);
+  gaussExp_e = 2*(e1+e2+timeResolution_e+PI_E);
 
   FDK_ASSERT( winSize < (1<<COEFF_LOOP_SCALE) );
 
@@ -1095,13 +1095,13 @@ static void FDKaacEnc_CalcGaussWindow(
   for( i=0; i<winSize; i++) {
 
     win[i] = fPow(
-            EULER_FIX,
-            EULER_SCALE,
+            EULER_M,
+            EULER_E,
             fMult(gaussExp_m, fPow2((i*FL2FXCONST_DBL(1.f/(float)(1<<COEFF_LOOP_SCALE)) + FL2FXCONST_DBL(.5f/(float)(1<<COEFF_LOOP_SCALE))))),
             gaussExp_e + 2*COEFF_LOOP_SCALE,
            &e1);
 
-    win[i] = scaleValue(win[i], e1);
+    win[i] = scaleValueSaturate(win[i], e1);
   }
 }
 
@@ -1157,7 +1157,10 @@ static INT FDKaacEnc_AutoToParcor(
     workBuffer++;
   }
 
-  tmp = fMult((FIXP_DBL)((LONG)TNS_PREDGAIN_SCALE<<21), fDivNorm(autoCorr_0, input[0], &scale));
+  tmp = fMult((FIXP_DBL)((LONG)TNS_PREDGAIN_SCALE<<21), fDivNorm(fAbs(autoCorr_0), fAbs(input[0]), &scale));
+  if ( fMultDiv2(autoCorr_0, input[0])<FL2FXCONST_DBL(0.0f) ) {
+    tmp = -tmp;
+  }
   predictionGain = (LONG)scaleValue(tmp,scale-21);
 
   return (predictionGain);
diff --git a/libAACenc/src/grp_data.cpp b/libAACenc/src/grp_data.cpp
index 03d4976..4355295 100644
--- a/libAACenc/src/grp_data.cpp
+++ b/libAACenc/src/grp_data.cpp
@@ -94,6 +94,10 @@ amm-info@iis.fraunhofer.de
 * this routine does not work in-place
 */
 
+static inline FIXP_DBL nrgAddSaturate(const FIXP_DBL a, const FIXP_DBL b) {
+  return ( (a>=(FIXP_DBL)MAXVAL_DBL-b) ? (FIXP_DBL)MAXVAL_DBL : (a + b) );
+}
+
 void
 FDKaacEnc_groupShortData(FIXP_DBL      *mdctSpectrum,     /* in-out                           */
                SFB_THRESHOLD *sfbThreshold,     /* in-out                           */
@@ -177,7 +181,7 @@ FDKaacEnc_groupShortData(FIXP_DBL      *mdctSpectrum,     /* in-out
       FIXP_DBL thresh = sfbThreshold->Short[wnd][sfb];
       for (j=1; j<groupLen[grp]; j++)
       {
-        thresh += sfbThreshold->Short[wnd+j][sfb];
+        thresh = nrgAddSaturate(thresh, sfbThreshold->Short[wnd+j][sfb]);
       }
       sfbThreshold->Long[i++] = thresh;
     }
@@ -195,7 +199,7 @@ FDKaacEnc_groupShortData(FIXP_DBL      *mdctSpectrum,     /* in-out
       FIXP_DBL energy = sfbEnergy->Short[wnd][sfb];
       for (j=1; j<groupLen[grp]; j++)
       {
-        energy += sfbEnergy->Short[wnd+j][sfb];
+        energy = nrgAddSaturate(energy, sfbEnergy->Short[wnd+j][sfb]);
       }
       sfbEnergy->Long[i++] = energy;
     }
@@ -213,7 +217,7 @@ FDKaacEnc_groupShortData(FIXP_DBL      *mdctSpectrum,     /* in-out
       FIXP_DBL energy = sfbEnergyMS->Short[wnd][sfb];
       for (j=1; j<groupLen[grp]; j++)
       {
-        energy += sfbEnergyMS->Short[wnd+j][sfb];
+        energy = nrgAddSaturate(energy, sfbEnergyMS->Short[wnd+j][sfb]);
       }
       sfbEnergyMS->Long[i++] = energy;
     }
@@ -231,7 +235,7 @@ FDKaacEnc_groupShortData(FIXP_DBL      *mdctSpectrum,     /* in-out
       FIXP_DBL energy = sfbSpreadEnergy->Short[wnd][sfb];
       for (j=1; j<groupLen[grp]; j++)
       {
-         energy += sfbSpreadEnergy->Short[wnd+j][sfb];
+         energy = nrgAddSaturate(energy, sfbSpreadEnergy->Short[wnd+j][sfb]);
       }
       sfbSpreadEnergy->Long[i++] = energy;
     }
diff --git a/libAACenc/src/qc_main.cpp b/libAACenc/src/qc_main.cpp
index 749398a..d7e76c7 100644
--- a/libAACenc/src/qc_main.cpp
+++ b/libAACenc/src/qc_main.cpp
@@ -797,7 +797,7 @@ AAC_ENCODER_ERROR FDKaacEnc_QCMain(QC_STATE* RESTRICT         hQC,
 {
   int i, c;
   AAC_ENCODER_ERROR ErrorStatus = AAC_ENC_OK;
-  INT avgTotalDynBits = 0; /* maximal allowd dynamic bits for all frames */
+  INT avgTotalDynBits = 0; /* maximal allowed dynamic bits for all frames */
   INT totalAvailableBits = 0;
   INT nSubFrames = 1;
 
@@ -1092,7 +1092,7 @@ AAC_ENCODER_ERROR FDKaacEnc_QCMain(QC_STATE* RESTRICT         hQC,
                 int sumBitsConsumedTotal = FDKaacEnc_getTotalConsumedBits(qcOut, qcElement, cm, hQC->globHdrBits, nSubFrames);
 
                 /* in all frames are valid dynamic bits */
-                if (sumBitsConsumedTotal < totalAvailableBits && (decreaseBitConsumption==1) && checkMinFrameBitsDemand(qcOut,hQC->minBitsPerFrame,nSubFrames)
+                if ( ((sumBitsConsumedTotal < totalAvailableBits) || qcOut[c]->usedDynBits==0) && (decreaseBitConsumption==1) && checkMinFrameBitsDemand(qcOut,hQC->minBitsPerFrame,nSubFrames)
                       /*()*/  )
                 {
                     quantizationDone = 1; /* exit bit adjustment */
@@ -1365,42 +1365,54 @@ AAC_ENCODER_ERROR FDKaacEnc_FinalizeBitConsumption(CHANNEL_MAPPING *cm,
   QC_OUT_EXTENSION fillExtPayload;
   INT totFillBits, alignBits;
 
-  {
-    int exactTpBits;
-    int max_iter = 3;
+  /* Get total consumed bits in AU */
+  qcOut->totalBits = qcOut->staticBits + qcOut->usedDynBits  + qcOut->totFillBits +
+                     qcOut->elementExtBits + qcOut->globalExtBits;
 
-    /* Get total consumed bits in AU */
-    qcOut->totalBits = qcOut->staticBits + qcOut->usedDynBits  + qcOut->totFillBits +
-                       qcOut->elementExtBits + qcOut->globalExtBits;
+  if (qcKernel->bitrateMode==QCDATA_BR_MODE_CBR) {
 
     /* Now we can get the exact transport bit amount, and hopefully it is equal to the estimated value */
-    exactTpBits = transportEnc_GetStaticBits(hTpEnc, qcOut->totalBits);
+    INT exactTpBits = transportEnc_GetStaticBits(hTpEnc, qcOut->totalBits);
 
     if (exactTpBits != qcKernel->globHdrBits) {
       INT diffFillBits = 0;
 
+      /* How many bits can be taken by bitreservoir */
+      const INT bitresSpace = qcKernel->bitResTotMax - (qcKernel->bitResTot + (qcOut->grantedDynBits - (qcOut->usedDynBits + qcOut->totFillBits) ) );
+
       /* Number of bits which can be moved to bitreservoir. */
-      INT bitsToBitres = qcKernel->globHdrBits - exactTpBits;
+      const INT bitsToBitres = qcKernel->globHdrBits - exactTpBits;
+      FDK_ASSERT(bitsToBitres>=0); /* is always positive */
 
-      if (bitsToBitres>0) {
-        /* if bitreservoir can not take all bits, move ramaining bits to fillbits */
-        diffFillBits = FDKmax(0, bitsToBitres - (qcKernel->bitResTotMax-qcKernel->bitResTot));
-      }
-      else if (bitsToBitres<0) {
-        /* if bits mus be taken from bitreservoir, reduce fillbits first. */
-        diffFillBits = (FDKmax(FDKmax(bitsToBitres, -qcKernel->bitResTot), -qcOut->totFillBits));
-      }
+      /* If bitreservoir can not take all bits, move ramaining bits to fillbits */
+      diffFillBits = FDKmax(0, bitsToBitres - bitresSpace);
+
+      /* Assure previous alignment */
+      diffFillBits = (diffFillBits+7)&~7;
 
-      diffFillBits = (diffFillBits+7)&~7; /* assure previous alignment */
+      /* Move as many bits as possible to bitreservoir */
+      qcKernel->bitResTot    += (bitsToBitres-diffFillBits);
 
+      /* Write remaing bits as fill bits */
       qcOut->totFillBits     += diffFillBits;
       qcOut->totalBits       += diffFillBits;
       qcOut->grantedDynBits  += diffFillBits;
 
-      /* new header bits */
+      /* Get new header bits */
       qcKernel->globHdrBits = transportEnc_GetStaticBits(hTpEnc, qcOut->totalBits);
+
+      if (qcKernel->globHdrBits != exactTpBits) {
+        /* In previous step, fill bits and corresponding total bits were changed when bitreservoir was completely filled.
+           Now we can take the too much taken bits caused by header overhead from bitreservoir.
+         */
+        qcKernel->bitResTot -= (qcKernel->globHdrBits - exactTpBits);
+      }
     }
-  }
+
+  } /* MODE_CBR */
+
+  /* Update exact number of consumed header bits. */
+  qcKernel->globHdrBits = transportEnc_GetStaticBits(hTpEnc, qcOut->totalBits);
 
   /* Save total fill bits and distribut to alignment and fill bits */
   totFillBits = qcOut->totFillBits;
diff --git a/libAACenc/src/quantize.cpp b/libAACenc/src/quantize.cpp
index dc85a6d..a1698a8 100644
--- a/libAACenc/src/quantize.cpp
+++ b/libAACenc/src/quantize.cpp
@@ -127,10 +127,7 @@ static void FDKaacEnc_quantizeLines(INT      gain,
       accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
       totalShift = (16-4)-(3*(totalShift>>2));
       FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
-      if (totalShift < 32)
-          accu>>=totalShift;
-      else
-          accu = 0;
+      accu>>=totalShift;
       quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16)));
     }
     else if(accu > FL2FXCONST_DBL(0.0f))
@@ -143,10 +140,7 @@ static void FDKaacEnc_quantizeLines(INT      gain,
       accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
       totalShift = (16-4)-(3*(totalShift>>2));
       FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
-      if (totalShift < 32)
-          accu>>=totalShift;
-      else
-          accu = 0;
+      accu>>=totalShift;
       quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16));
     }
     else
@@ -319,6 +313,9 @@ FIXP_DBL FDKaacEnc_calcSfbDist(FIXP_DBL *mdctSpectrum,
                  &mdctSpectrum[i],
                  &quantSpectrum[i]);
 
+    if (fAbs(quantSpectrum[i])>MAX_QUANT) {
+      return FL2FXCONST_DBL(0.0f);
+    }
     /* inverse quantization */
     FDKaacEnc_invQuantizeLines(gain,1,&quantSpectrum[i],&invQuantSpec);
 
@@ -361,15 +358,22 @@ void FDKaacEnc_calcSfbQuantEnergyAndDist(FIXP_DBL *mdctSpectrum,
   FIXP_DBL invQuantSpec;
   FIXP_DBL diff;
 
-  *en   = FL2FXCONST_DBL(0.0f);
-  *dist = FL2FXCONST_DBL(0.0f);
+  FIXP_DBL energy = FL2FXCONST_DBL(0.0f);
+  FIXP_DBL distortion = FL2FXCONST_DBL(0.0f);
 
   for (i=0; i<noOfLines; i++) {
+
+    if (fAbs(quantSpectrum[i])>MAX_QUANT) {
+      *en   = FL2FXCONST_DBL(0.0f);
+      *dist = FL2FXCONST_DBL(0.0f);
+      return;
+    }
+
     /* inverse quantization */
     FDKaacEnc_invQuantizeLines(gain,1,&quantSpectrum[i],&invQuantSpec);
 
     /* energy */
-    *en += fPow2(invQuantSpec);
+    energy += fPow2(invQuantSpec);
 
     /* dist */
     diff = fixp_abs(fixp_abs(invQuantSpec) - fixp_abs(mdctSpectrum[i]>>1));
@@ -382,10 +386,10 @@ void FDKaacEnc_calcSfbQuantEnergyAndDist(FIXP_DBL *mdctSpectrum,
 
     diff = scaleValue(diff, -scale);
 
-    *dist += diff;
+    distortion += diff;
   }
 
-  *en   = CalcLdData(*en)+FL2FXCONST_DBL(0.03125f);
-  *dist = CalcLdData(*dist);
+  *en   = CalcLdData(energy)+FL2FXCONST_DBL(0.03125f);
+  *dist = CalcLdData(distortion);
 }