FDKv2 ubsan patches

Bug: 80053205
Test: see bug for repro with FB "wow"
   atest DecoderTestAacDrc

Fix signed integer overflows in CLpc_SynthesisLattice()

Change-Id: Icbddfcc8c5fc73382ae5bf8c2a7703802c688e06

Fix signed integer overflows in imlt

Change-Id: I687834fca2f1aab6210ed9862576b4f38fcdeb24

Fix overflow in addLowbandEnergies()

Change-Id: Iaa9fdf9deb49c33ec6ca7ed3081c4ddaa920e9aa

Concealment fix for audio frames containing acelp components

Change-Id: Ibe5e83a6efa75a48f729984a161a76b826878f4e

Fix out-of-bounds access in PS concealment

Change-Id: I08809a03a40d1feaf00e41278db314d67e1efe88

Fix potential memory leak in setup of qmf domain

Change-Id: Id9fc2448354dc7f1b439469128407305efa3def2

Reject channel config 13

Change-Id: Idf5236f6cd054df994e69c9c972c97f6768cf9e5

Fix unsigned integer overflow in configExtension()

Change-Id: I8a1668810b85e6237c3892891444ff08f04b019b

Fix unsigned integer overflow in CAacDecoder_DecodeFrame()

Change-Id: I79678c571690178e6c37680f70a9b94dd3cbc439

Fix unsigned integer overflow in aacDecoder_UpdateBitStreamCounters()

Change-Id: I3bff959da9f53fabb18cd0ae6c260e6256194526

Fix unsigned integer overflow in transportDec_readStream()

Change-Id: I6a6f9f4acaa32fae0b5de9641f8787bbc7f8286b

5 files changed, 105 insertions, 102 deletions

diff --git a/libFDK/include/cplx_mul.h b/libFDK/include/cplx_mul.h
index 9c5da57..eb1afce 100644
--- a/libFDK/include/cplx_mul.h
+++ b/libFDK/include/cplx_mul.h
@@ -260,26 +260,6 @@ inline void cplxMult(FIXP_DBL *c_Re, FIXP_DBL *c_Im, const FIXP_DBL a_Re,
 }
 #endif
 
-#if !defined(FUNCTION_cplxMult_nIm)
-#define FUNCTION_cplxMult_nIm
-
-/* Same as cplxMult, but
-   a_Im must be negated, when used
-   c_re must be negated, when output
-*/
-inline void cplxMult_nIm(FIXP_DBL *c_Re, FIXP_DBL *c_Im, const FIXP_DBL a_Re,
-                         const FIXP_DBL a_Im, const FIXP_SPK w) {
-  *c_Re = -(fMult(a_Re, w.v.re) + fMult(a_Im, w.v.im));
-  *c_Im = fMult(a_Re, w.v.im) - fMult(a_Im, w.v.re);
-}
-
-inline void cplxMult_nIm(FIXP_DBL *c_Re, FIXP_DBL *c_Im, const FIXP_DBL a_Re,
-                         const FIXP_DBL a_Im, const FIXP_DPK w) {
-  *c_Re = -(fMult(a_Re, w.v.re) + fMult(a_Im, w.v.im));
-  *c_Im = fMult(a_Re, w.v.im) - fMult(a_Im, w.v.re);
-}
-#endif
-
   /* #############################################################################
    */
 
diff --git a/libFDK/include/mdct.h b/libFDK/include/mdct.h
index e671da9..1382374 100644
--- a/libFDK/include/mdct.h
+++ b/libFDK/include/mdct.h
@@ -119,6 +119,7 @@ amm-info@iis.fraunhofer.de
 #define IMDCT_SCALE(x) SATURATE_LEFT_SHIFT(x, -MDCT_OUTPUT_SCALE, PCM_OUT_BITS)
 #endif
 #define IMDCT_SCALE_DBL(x) (FIXP_DBL)(x)
+#define IMDCT_SCALE_DBL_LSH1(x) SATURATE_LEFT_SHIFT_ALT((x), 1, DFRACT_BITS)
 
 #define MLT_FLAG_CURR_ALIAS_SYMMETRY 1
 
diff --git a/libFDK/src/FDK_lpc.cpp b/libFDK/src/FDK_lpc.cpp
index 2ba7707..7d7e691 100644
--- a/libFDK/src/FDK_lpc.cpp
+++ b/libFDK/src/FDK_lpc.cpp
@@ -184,12 +184,19 @@ void CLpc_SynthesisLattice(FIXP_DBL *signal, const int signal_size,
   for (i = signal_size; i != 0; i--) {
     FIXP_DBL *pState = state + order - 1;
     const FIXP_DBL *pCoeff = coeff + order - 1;
-    FIXP_DBL tmp;
+    FIXP_DBL tmp, accu;
+
+    accu =
+        fMultSubDiv2(scaleValue(*pSignal, signal_e - 1), *pCoeff--, *pState--);
+    tmp = SATURATE_LEFT_SHIFT_ALT(accu, 1, DFRACT_BITS);
 
-    tmp = scaleValue(*pSignal, signal_e) - fMult(*pCoeff--, *pState--);
     for (j = order - 1; j != 0; j--) {
-      tmp = tmp - fMult(pCoeff[0], pState[0]);
-      pState[1] = pState[0] + fMult(*pCoeff--, tmp);
+      accu = fMultSubDiv2(tmp >> 1, pCoeff[0], pState[0]);
+      tmp = SATURATE_LEFT_SHIFT_ALT(accu, 1, DFRACT_BITS);
+
+      accu = fMultAddDiv2(pState[0] >> 1, *pCoeff--, tmp);
+      pState[1] = SATURATE_LEFT_SHIFT_ALT(accu, 1, DFRACT_BITS);
+
       pState--;
     }
 
diff --git a/libFDK/src/FDK_qmf_domain.cpp b/libFDK/src/FDK_qmf_domain.cpp
index 4b78931..043a372 100644
--- a/libFDK/src/FDK_qmf_domain.cpp
+++ b/libFDK/src/FDK_qmf_domain.cpp
@@ -274,17 +274,28 @@ static int FDK_QmfDomain_AllocatePersistentMemory(HANDLE_FDK_QMF_DOMAIN qd) {
     size = gc->nBandsAnalysis * 10;
     if (size > 0) {
       if (gc->nBandsAnalysis == QMF_DOMAIN_ANALYSIS_QMF_BANDS_16) {
-        if (NULL == (qd->QmfDomainIn[ch].pAnaQmfStates = GetAnaQmfStates16(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].pAnaQmfStates == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].pAnaQmfStates = GetAnaQmfStates16(ch)))
+            goto bail;
+        }
       } else if (gc->nBandsAnalysis == QMF_DOMAIN_ANALYSIS_QMF_BANDS_24) {
-        if (NULL == (qd->QmfDomainIn[ch].pAnaQmfStates = GetAnaQmfStates24(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].pAnaQmfStates == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].pAnaQmfStates = GetAnaQmfStates24(ch)))
+            goto bail;
+        }
       } else if (gc->nBandsAnalysis == QMF_DOMAIN_ANALYSIS_QMF_BANDS_32) {
-        if (NULL == (qd->QmfDomainIn[ch].pAnaQmfStates = GetAnaQmfStates32(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].pAnaQmfStates == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].pAnaQmfStates = GetAnaQmfStates32(ch)))
+            goto bail;
+        }
       } else {
-        if (NULL == (qd->QmfDomainIn[ch].pAnaQmfStates = GetAnaQmfStates(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].pAnaQmfStates == NULL) {
+          if (NULL == (qd->QmfDomainIn[ch].pAnaQmfStates = GetAnaQmfStates(ch)))
+            goto bail;
+        }
       }
     } else {
       qd->QmfDomainIn[ch].pAnaQmfStates = NULL;
@@ -293,20 +304,36 @@ static int FDK_QmfDomain_AllocatePersistentMemory(HANDLE_FDK_QMF_DOMAIN qd) {
     size = gc->nQmfOvTimeSlots + gc->nQmfTimeSlots;
     if (size > 0) {
       if (gc->nQmfTimeSlots == QMF_DOMAIN_TIMESLOTS_16) {
-        if (NULL == (qd->QmfDomainIn[ch].hQmfSlotsReal = GetQmfSlotsReal16(ch)))
-          goto bail;
-        if (NULL == (qd->QmfDomainIn[ch].hQmfSlotsImag = GetQmfSlotsImag16(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].hQmfSlotsReal == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].hQmfSlotsReal = GetQmfSlotsReal16(ch)))
+            goto bail;
+        }
+        if (qd->QmfDomainIn[ch].hQmfSlotsImag == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].hQmfSlotsImag = GetQmfSlotsImag16(ch)))
+            goto bail;
+        }
       } else if (gc->nQmfTimeSlots == QMF_DOMAIN_TIMESLOTS_32) {
-        if (NULL == (qd->QmfDomainIn[ch].hQmfSlotsReal = GetQmfSlotsReal32(ch)))
-          goto bail;
-        if (NULL == (qd->QmfDomainIn[ch].hQmfSlotsImag = GetQmfSlotsImag32(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].hQmfSlotsReal == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].hQmfSlotsReal = GetQmfSlotsReal32(ch)))
+            goto bail;
+        }
+        if (qd->QmfDomainIn[ch].hQmfSlotsImag == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].hQmfSlotsImag = GetQmfSlotsImag32(ch)))
+            goto bail;
+        }
       } else {
-        if (NULL == (qd->QmfDomainIn[ch].hQmfSlotsReal = GetQmfSlotsReal(ch)))
-          goto bail;
-        if (NULL == (qd->QmfDomainIn[ch].hQmfSlotsImag = GetQmfSlotsImag(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].hQmfSlotsReal == NULL) {
+          if (NULL == (qd->QmfDomainIn[ch].hQmfSlotsReal = GetQmfSlotsReal(ch)))
+            goto bail;
+        }
+        if (qd->QmfDomainIn[ch].hQmfSlotsImag == NULL) {
+          if (NULL == (qd->QmfDomainIn[ch].hQmfSlotsImag = GetQmfSlotsImag(ch)))
+            goto bail;
+        }
       }
     } else {
       qd->QmfDomainIn[ch].hQmfSlotsReal = NULL;
@@ -316,17 +343,23 @@ static int FDK_QmfDomain_AllocatePersistentMemory(HANDLE_FDK_QMF_DOMAIN qd) {
     size = gc->nQmfOvTimeSlots * gc->nQmfProcBands * CMPLX_MOD;
     if (size > 0) {
       if (gc->nQmfOvTimeSlots == QMF_DOMAIN_OV_TIMESLOTS_16) {
-        if (NULL ==
-            (qd->QmfDomainIn[ch].pOverlapBuffer = GetQmfOverlapBuffer16(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].pOverlapBuffer == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].pOverlapBuffer = GetQmfOverlapBuffer16(ch)))
+            goto bail;
+        }
       } else if (gc->nQmfOvTimeSlots == QMF_DOMAIN_OV_TIMESLOTS_32) {
-        if (NULL ==
-            (qd->QmfDomainIn[ch].pOverlapBuffer = GetQmfOverlapBuffer32(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].pOverlapBuffer == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].pOverlapBuffer = GetQmfOverlapBuffer32(ch)))
+            goto bail;
+        }
       } else {
-        if (NULL ==
-            (qd->QmfDomainIn[ch].pOverlapBuffer = GetQmfOverlapBuffer(ch)))
-          goto bail;
+        if (qd->QmfDomainIn[ch].pOverlapBuffer == NULL) {
+          if (NULL ==
+              (qd->QmfDomainIn[ch].pOverlapBuffer = GetQmfOverlapBuffer(ch)))
+            goto bail;
+        }
       }
     } else {
       qd->QmfDomainIn[ch].pOverlapBuffer = NULL;
@@ -336,8 +369,10 @@ static int FDK_QmfDomain_AllocatePersistentMemory(HANDLE_FDK_QMF_DOMAIN qd) {
   for (ch = 0; ch < gc->nOutputChannels; ch++) {
     int size = gc->nBandsSynthesis * 9;
     if (size > 0) {
-      if (NULL == (qd->QmfDomainOut[ch].pSynQmfStates = GetSynQmfStates(ch)))
-        goto bail;
+      if (qd->QmfDomainOut[ch].pSynQmfStates == NULL) {
+        if (NULL == (qd->QmfDomainOut[ch].pSynQmfStates = GetSynQmfStates(ch)))
+          goto bail;
+      }
     } else {
       qd->QmfDomainOut[ch].pSynQmfStates = NULL;
     }
diff --git a/libFDK/src/mdct.cpp b/libFDK/src/mdct.cpp
index 6a6604c..d697cfb 100644
--- a/libFDK/src/mdct.cpp
+++ b/libFDK/src/mdct.cpp
@@ -541,11 +541,16 @@ INT imlt_block(H_MDCT hMdct, FIXP_DBL *output, FIXP_DBL *spectrum,
      */
     /* and de-scale current spectrum signal (time domain, no yet windowed) */
     if (gain != (FIXP_DBL)0) {
-      scaleValuesWithFactor(pSpec, gain, tl, scalefactor[w] + specShiftScale);
-    } else {
-      int loc_scale = scalefactor[w] + specShiftScale;
+      for (i = 0; i < tl; i++) {
+        pSpec[i] = fMult(pSpec[i], gain);
+      }
+    }
+
+    {
+      int loc_scale =
+          fixmin_I(scalefactor[w] + specShiftScale, (INT)DFRACT_BITS - 1);
       DWORD_ALIGNED(pSpec);
-      scaleValues(pSpec, tl, loc_scale);
+      scaleValuesSaturate(pSpec, tl, loc_scale);
     }
 
     if (noOutSamples <= nrSamples) {
@@ -614,59 +619,34 @@ INT imlt_block(H_MDCT hMdct, FIXP_DBL *output, FIXP_DBL *spectrum,
         if (!hMdct->pAsymOvlp) {
           for (i = 0; i < fl / 2; i++) {
             FIXP_DBL x0, x1;
-#ifdef FUNCTION_cplxMult_nIm
-            /* This macro negates 4th parameter (*pOvl--) */
-            /* and inverts the sign of result x1          */
-
-            /* This subroutine calculates the two output segments (C,D) from the
-            two availabe  DCT IV data blocks, namely, (-D-Cr,A-Br) and
-            (-F-Er,C-Dr). "pOvl" is the pointer to the overlap block and points
-            to the end of the (-D-Cr) part of the overlap buffer (-D-Cr,A-Br).
-            It points to the end of the (-D-Cr) because it will read this part
-            in a flipped order. "pCurr" is the pointer to the current block
-            (-F-Er,C-Dr) and points to the beginning of the (C-Dr) block,
-            because this block will be read consequitively. "pWindow" is a
-            pointer to the used window coefficients. In pointer "x1" we get the
-            already computed from the function "Dr" segment. In pointer "x0" we
-            get the "C" segment. Since we have to output them sequentially the
-            "x0" pointer points to the beginnig of the output buffer (X,X), and
-            pointer "x1" points to the end of the output buffer (X,X). When we
-            get the output of the cplxMult_nIm function we write it sequentially
-            in the output buffer from the left to right ("x0"=>C) and right to
-            left ("x1"=>Dr) implementing flipping. At the end we get an output
-            in the form (C,D).      */
-            cplxMult_nIm(&x1, &x0, *pCurr++, *pOvl--, pWindow[i]);
-            *pOut0++ = IMDCT_SCALE_DBL(x0);
-            *pOut1-- = IMDCT_SCALE_DBL(x1);
-#else
-            cplxMult(&x1, &x0, *pCurr++, -*pOvl--, pWindow[i]);
-            *pOut0 = IMDCT_SCALE_DBL(x0);
-            *pOut1 = IMDCT_SCALE_DBL(-x1);
+            cplxMultDiv2(&x1, &x0, *pCurr++, -*pOvl--, pWindow[i]);
+            *pOut0 = IMDCT_SCALE_DBL_LSH1(x0);
+            *pOut1 = IMDCT_SCALE_DBL_LSH1(-x1);
             pOut0++;
             pOut1--;
-#endif /* #ifdef FUNCTION_cplxMult_nIm */
           }
         } else {
           FIXP_DBL *pAsymOvl = hMdct->pAsymOvlp + fl / 2 - 1;
           for (i = 0; i < fl / 2; i++) {
             FIXP_DBL x0, x1;
-            x1 = -fMult(*pCurr, pWindow[i].v.re) +
-                 fMult(*pAsymOvl, pWindow[i].v.im);
-            x0 = fMult(*pCurr, pWindow[i].v.im) - fMult(*pOvl, pWindow[i].v.re);
+            x1 = -fMultDiv2(*pCurr, pWindow[i].v.re) +
+                 fMultDiv2(*pAsymOvl, pWindow[i].v.im);
+            x0 = fMultDiv2(*pCurr, pWindow[i].v.im) -
+                 fMultDiv2(*pOvl, pWindow[i].v.re);
             pCurr++;
             pOvl--;
             pAsymOvl--;
-            *pOut0++ = IMDCT_SCALE_DBL(x0);
-            *pOut1-- = IMDCT_SCALE_DBL(x1);
+            *pOut0++ = IMDCT_SCALE_DBL_LSH1(x0);
+            *pOut1-- = IMDCT_SCALE_DBL_LSH1(x1);
           }
           hMdct->pAsymOvlp = NULL;
         }
       } else { /* prevAliasingSymmetry == 1 */
         for (i = 0; i < fl / 2; i++) {
           FIXP_DBL x0, x1;
-          cplxMult(&x1, &x0, *pCurr++, -*pOvl--, pWindow[i]);
-          *pOut0 = IMDCT_SCALE_DBL(x0);
-          *pOut1 = IMDCT_SCALE_DBL(x1);
+          cplxMultDiv2(&x1, &x0, *pCurr++, -*pOvl--, pWindow[i]);
+          *pOut0 = IMDCT_SCALE_DBL_LSH1(x0);
+          *pOut1 = IMDCT_SCALE_DBL_LSH1(x1);
           pOut0++;
           pOut1--;
         }
@@ -675,18 +655,18 @@ INT imlt_block(H_MDCT hMdct, FIXP_DBL *output, FIXP_DBL *spectrum,
       if (hMdct->prevAliasSymmetry == 0) {
         for (i = 0; i < fl / 2; i++) {
           FIXP_DBL x0, x1;
-          cplxMult(&x1, &x0, *pCurr++, *pOvl--, pWindow[i]);
-          *pOut0 = IMDCT_SCALE_DBL(x0);
-          *pOut1 = IMDCT_SCALE_DBL(-x1);
+          cplxMultDiv2(&x1, &x0, *pCurr++, *pOvl--, pWindow[i]);
+          *pOut0 = IMDCT_SCALE_DBL_LSH1(x0);
+          *pOut1 = IMDCT_SCALE_DBL_LSH1(-x1);
           pOut0++;
           pOut1--;
         }
       } else { /* prevAliasingSymmetry == 1 */
         for (i = 0; i < fl / 2; i++) {
           FIXP_DBL x0, x1;
-          cplxMult(&x1, &x0, *pCurr++, *pOvl--, pWindow[i]);
-          *pOut0 = IMDCT_SCALE_DBL(x0);
-          *pOut1 = IMDCT_SCALE_DBL(x1);
+          cplxMultDiv2(&x1, &x0, *pCurr++, *pOvl--, pWindow[i]);
+          *pOut0 = IMDCT_SCALE_DBL_LSH1(x0);
+          *pOut1 = IMDCT_SCALE_DBL_LSH1(x1);
           pOut0++;
           pOut1--;
         }