Introduce scaling in FDKhybridSynthesisApply() to avoid potential signed integer overflow.

Bug: 146937225
Test: atest DecoderTestXheAac ; atest DecoderTestAacDrc
Change-Id: Ifa3d157a842828c2a66694d95dad5d4992ff6d87

1 files changed, 18 insertions, 16 deletions

diff --git a/libFDK/src/FDK_hybrid.cpp b/libFDK/src/FDK_hybrid.cpp
index b661f82..08d32a8 100644
--- a/libFDK/src/FDK_hybrid.cpp
+++ b/libFDK/src/FDK_hybrid.cpp
@@ -1,7 +1,7 @@
 /* -----------------------------------------------------------------------------
 Software License for The Fraunhofer FDK AAC Codec Library for Android
 
-© Copyright  1995 - 2018 Fraunhofer-Gesellschaft zur Förderung der angewandten
+© Copyright  1995 - 2019 Fraunhofer-Gesellschaft zur Förderung der angewandten
 Forschung e.V. All rights reserved.
 
  1.    INTRODUCTION
@@ -142,11 +142,12 @@ amm-info@iis.fraunhofer.de
   } /* How to arrange the packed values. */
 
 struct FDK_HYBRID_SETUP {
-  UCHAR nrQmfBands;   /*!< Number of QMF bands to be converted to hybrid. */
-  UCHAR nHybBands[3]; /*!< Number of Hybrid bands generated by nrQmfBands. */
-  SCHAR kHybrid[3];   /*!< Filter configuration of each QMF band. */
-  UCHAR protoLen;     /*!< Prototype filter length. */
-  UCHAR filterDelay;  /*!< Delay caused by hybrid filter. */
+  UCHAR nrQmfBands;     /*!< Number of QMF bands to be converted to hybrid. */
+  UCHAR nHybBands[3];   /*!< Number of Hybrid bands generated by nrQmfBands. */
+  UCHAR synHybScale[3]; /*!< Headroom needed in hybrid synthesis filterbank. */
+  SCHAR kHybrid[3];     /*!< Filter configuration of each QMF band. */
+  UCHAR protoLen;       /*!< Prototype filter length. */
+  UCHAR filterDelay;    /*!< Delay caused by hybrid filter. */
   const INT
       *pReadIdxTable; /*!< Helper table to access input data ringbuffer. */
 };
@@ -156,12 +157,12 @@ static const INT ringbuffIdxTab[2 * 13] = {0, 1,  2,  3,  4, 5,  6,  7, 8,
                                            9, 10, 11, 12, 0, 1,  2,  3, 4,
                                            5, 6,  7,  8,  9, 10, 11, 12};
 
-static const FDK_HYBRID_SETUP setup_3_16 = {3,  {8, 4, 4},    {8, 4, 4},
-                                            13, (13 - 1) / 2, ringbuffIdxTab};
-static const FDK_HYBRID_SETUP setup_3_12 = {3,  {8, 2, 2},    {8, 2, 2},
-                                            13, (13 - 1) / 2, ringbuffIdxTab};
-static const FDK_HYBRID_SETUP setup_3_10 = {3,  {6, 2, 2},    {-8, -2, 2},
-                                            13, (13 - 1) / 2, ringbuffIdxTab};
+static const FDK_HYBRID_SETUP setup_3_16 = {
+    3, {8, 4, 4}, {4, 3, 3}, {8, 4, 4}, 13, (13 - 1) / 2, ringbuffIdxTab};
+static const FDK_HYBRID_SETUP setup_3_12 = {
+    3, {8, 2, 2}, {4, 2, 2}, {8, 2, 2}, 13, (13 - 1) / 2, ringbuffIdxTab};
+static const FDK_HYBRID_SETUP setup_3_10 = {
+    3, {6, 2, 2}, {3, 2, 2}, {-8, -2, 2}, 13, (13 - 1) / 2, ringbuffIdxTab};
 
 static const FIXP_HTP HybFilterCoef8[] = {
     HTCP(0x10000000, 0x00000000), HTCP(0x0df26407, 0xfa391882),
@@ -477,17 +478,18 @@ void FDKhybridSynthesisApply(HANDLE_FDK_SYN_HYB_FILTER hSynthesisHybFilter,
    */
   for (k = 0; k < nrQmfBandsLF; k++) {
     const int nHybBands = hSynthesisHybFilter->pSetup->nHybBands[k];
+    const int scale = hSynthesisHybFilter->pSetup->synHybScale[k];
 
     FIXP_DBL accu1 = FL2FXCONST_DBL(0.f);
     FIXP_DBL accu2 = FL2FXCONST_DBL(0.f);
 
     /* Perform hybrid filtering. */
     for (n = 0; n < nHybBands; n++) {
-      accu1 += pHybridReal[hybOffset + n];
-      accu2 += pHybridImag[hybOffset + n];
+      accu1 += pHybridReal[hybOffset + n] >> scale;
+      accu2 += pHybridImag[hybOffset + n] >> scale;
     }
-    pQmfReal[k] = accu1;
-    pQmfImag[k] = accu2;
+    pQmfReal[k] = SATURATE_LEFT_SHIFT(accu1, scale, DFRACT_BITS);
+    pQmfImag[k] = SATURATE_LEFT_SHIFT(accu2, scale, DFRACT_BITS);
 
     hybOffset += nHybBands;
   }