Prevent signed integer overflow in complex path of calc_qmfBuffer().

Bug: 186706541
Test: atest android.media.cts.DecoderTestAacFormat android.media.cts.DecoderTestXheAac android.media.cts.DecoderTestAacDrc
Change-Id: Iae0581ea983912bcf1bc6b3bdc2c0d8ee2f6248e

2 files changed, 31 insertions, 194 deletions

diff --git a/libSBRdec/src/arm/lpp_tran_arm.cpp b/libSBRdec/src/arm/lpp_tran_arm.cpp
deleted file mode 100644
index db1948f..0000000
--- a/libSBRdec/src/arm/lpp_tran_arm.cpp
+++ /dev/null
@@ -1,159 +0,0 @@
-/* -----------------------------------------------------------------------------
-Software License for The Fraunhofer FDK AAC Codec Library for Android
-
-© Copyright  1995 - 2018 Fraunhofer-Gesellschaft zur Förderung der angewandten
-Forschung e.V. All rights reserved.
-
- 1.    INTRODUCTION
-The Fraunhofer FDK AAC Codec Library for Android ("FDK AAC Codec") is software
-that implements the MPEG Advanced Audio Coding ("AAC") encoding and decoding
-scheme for digital audio. This FDK AAC Codec software is intended to be used on
-a wide variety of Android devices.
-
-AAC's HE-AAC and HE-AAC v2 versions are regarded as today's most efficient
-general perceptual audio codecs. AAC-ELD is considered the best-performing
-full-bandwidth communications codec by independent studies and is widely
-deployed. AAC has been standardized by ISO and IEC as part of the MPEG
-specifications.
-
-Patent licenses for necessary patent claims for the FDK AAC Codec (including
-those of Fraunhofer) may be obtained through Via Licensing
-(www.vialicensing.com) or through the respective patent owners individually for
-the purpose of encoding or decoding bit streams in products that are compliant
-with the ISO/IEC MPEG audio standards. Please note that most manufacturers of
-Android devices already license these patent claims through Via Licensing or
-directly from the patent owners, and therefore FDK AAC Codec software may
-already be covered under those patent licenses when it is used for those
-licensed purposes only.
-
-Commercially-licensed AAC software libraries, including floating-point versions
-with enhanced sound quality, are also available from Fraunhofer. Users are
-encouraged to check the Fraunhofer website for additional applications
-information and documentation.
-
-2.    COPYRIGHT LICENSE
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted without payment of copyright license fees provided that you
-satisfy the following conditions:
-
-You must retain the complete text of this software license in redistributions of
-the FDK AAC Codec or your modifications thereto in source code form.
-
-You must retain the complete text of this software license in the documentation
-and/or other materials provided with redistributions of the FDK AAC Codec or
-your modifications thereto in binary form. You must make available free of
-charge copies of the complete source code of the FDK AAC Codec and your
-modifications thereto to recipients of copies in binary form.
-
-The name of Fraunhofer may not be used to endorse or promote products derived
-from this library without prior written permission.
-
-You may not charge copyright license fees for anyone to use, copy or distribute
-the FDK AAC Codec software or your modifications thereto.
-
-Your modified versions of the FDK AAC Codec must carry prominent notices stating
-that you changed the software and the date of any change. For modified versions
-of the FDK AAC Codec, the term "Fraunhofer FDK AAC Codec Library for Android"
-must be replaced by the term "Third-Party Modified Version of the Fraunhofer FDK
-AAC Codec Library for Android."
-
-3.    NO PATENT LICENSE
-
-NO EXPRESS OR IMPLIED LICENSES TO ANY PATENT CLAIMS, including without
-limitation the patents of Fraunhofer, ARE GRANTED BY THIS SOFTWARE LICENSE.
-Fraunhofer provides no warranty of patent non-infringement with respect to this
-software.
-
-You may use this FDK AAC Codec software or modifications thereto only for
-purposes that are authorized by appropriate patent licenses.
-
-4.    DISCLAIMER
-
-This FDK AAC Codec software is provided by Fraunhofer on behalf of the copyright
-holders and contributors "AS IS" and WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES,
-including but not limited to the implied warranties of merchantability and
-fitness for a particular purpose. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
-CONTRIBUTORS BE LIABLE for any direct, indirect, incidental, special, exemplary,
-or consequential damages, including but not limited to procurement of substitute
-goods or services; loss of use, data, or profits, or business interruption,
-however caused and on any theory of liability, whether in contract, strict
-liability, or tort (including negligence), arising in any way out of the use of
-this software, even if advised of the possibility of such damage.
-
-5.    CONTACT INFORMATION
-
-Fraunhofer Institute for Integrated Circuits IIS
-Attention: Audio and Multimedia Departments - FDK AAC LL
-Am Wolfsmantel 33
-91058 Erlangen, Germany
-
-www.iis.fraunhofer.de/amm
-amm-info@iis.fraunhofer.de
------------------------------------------------------------------------------ */
-
-/**************************** SBR decoder library ******************************
-
-   Author(s):   Arthur Tritthart
-
-   Description: (ARM optimised) LPP transposer subroutines
-
-*******************************************************************************/
-
-#if defined(__arm__)
-
-#define FUNCTION_LPPTRANSPOSER_func1
-
-#ifdef FUNCTION_LPPTRANSPOSER_func1
-
-/* Note: This code requires only 43 cycles per iteration instead of 61 on
- * ARM926EJ-S */
-static void lppTransposer_func1(FIXP_DBL *lowBandReal, FIXP_DBL *lowBandImag,
-                                FIXP_DBL **qmfBufferReal,
-                                FIXP_DBL **qmfBufferImag, int loops, int hiBand,
-                                int dynamicScale, int descale, FIXP_SGL a0r,
-                                FIXP_SGL a0i, FIXP_SGL a1r, FIXP_SGL a1i,
-                                const int fPreWhitening,
-                                FIXP_DBL preWhiteningGain,
-                                int preWhiteningGains_sf) {
-  FIXP_DBL real1, real2, imag1, imag2, accu1, accu2;
-
-  real2 = lowBandReal[-2];
-  real1 = lowBandReal[-1];
-  imag2 = lowBandImag[-2];
-  imag1 = lowBandImag[-1];
-  for (int i = 0; i < loops; i++) {
-    accu1 = fMultDiv2(a0r, real1);
-    accu2 = fMultDiv2(a0i, imag1);
-    accu1 = fMultAddDiv2(accu1, a1r, real2);
-    accu2 = fMultAddDiv2(accu2, a1i, imag2);
-    real2 = fMultDiv2(a1i, real2);
-    accu1 = accu1 - accu2;
-    accu1 = accu1 >> dynamicScale;
-
-    accu2 = fMultAddDiv2(real2, a1r, imag2);
-    real2 = real1;
-    imag2 = imag1;
-    accu2 = fMultAddDiv2(accu2, a0i, real1);
-    real1 = lowBandReal[i];
-    accu2 = fMultAddDiv2(accu2, a0r, imag1);
-    imag1 = lowBandImag[i];
-    accu2 = accu2 >> dynamicScale;
-
-    accu1 <<= 1;
-    accu2 <<= 1;
-    accu1 += (real1 >> descale);
-    accu2 += (imag1 >> descale);
-    if (fPreWhitening) {
-      accu1 = scaleValueSaturate(fMultDiv2(accu1, preWhiteningGain),
-                                 preWhiteningGains_sf);
-      accu2 = scaleValueSaturate(fMultDiv2(accu2, preWhiteningGain),
-                                 preWhiteningGains_sf);
-    }
-    qmfBufferReal[i][hiBand] = accu1;
-    qmfBufferImag[i][hiBand] = accu2;
-  }
-}
-#endif /* #ifdef FUNCTION_LPPTRANSPOSER_func1 */
-
-#endif /* __arm__ */
diff --git a/libSBRdec/src/lpp_tran.cpp b/libSBRdec/src/lpp_tran.cpp
index 93e1158..bdbafa6 100644
--- a/libSBRdec/src/lpp_tran.cpp
+++ b/libSBRdec/src/lpp_tran.cpp
@@ -132,10 +132,6 @@ amm-info@iis.fraunhofer.de
 
 #include "HFgen_preFlat.h"
 
-#if defined(__arm__)
-#include "arm/lpp_tran_arm.cpp"
-#endif
-
 #define LPC_SCALE_FACTOR 2
 
 /*!
@@ -771,45 +767,45 @@ void lppTransposer(
       } else { /* bw <= 0 */
 
         if (!useLP) {
-          int descale =
-              fixMin(DFRACT_BITS - 1, (LPC_SCALE_FACTOR + dynamicScale));
-#ifdef FUNCTION_LPPTRANSPOSER_func1
-          lppTransposer_func1(
-              lowBandReal + LPC_ORDER + startSample,
-              lowBandImag + LPC_ORDER + startSample,
-              qmfBufferReal + startSample, qmfBufferImag + startSample,
-              stopSample - startSample, (int)hiBand, dynamicScale, descale, a0r,
-              a0i, a1r, a1i, fPreWhitening, preWhiteningGains[loBand],
-              preWhiteningGains_exp[loBand] + 1);
-#else
+          const int dynscale = fixMax(0, dynamicScale - 2) + 1;
+          const int rescale = -fixMin(0, dynamicScale - 2) + 1;
+          const int descale = fixMin(DFRACT_BITS - 1,
+                                     LPC_SCALE_FACTOR + dynamicScale + rescale);
+
           for (i = startSample; i < stopSample; i++) {
             FIXP_DBL accu1, accu2;
 
-            accu1 = (fMultDiv2(a0r, lowBandReal[LPC_ORDER + i - 1]) -
-                     fMultDiv2(a0i, lowBandImag[LPC_ORDER + i - 1]) +
-                     fMultDiv2(a1r, lowBandReal[LPC_ORDER + i - 2]) -
-                     fMultDiv2(a1i, lowBandImag[LPC_ORDER + i - 2])) >>
-                    dynamicScale;
-            accu2 = (fMultDiv2(a0i, lowBandReal[LPC_ORDER + i - 1]) +
-                     fMultDiv2(a0r, lowBandImag[LPC_ORDER + i - 1]) +
-                     fMultDiv2(a1i, lowBandReal[LPC_ORDER + i - 2]) +
-                     fMultDiv2(a1r, lowBandImag[LPC_ORDER + i - 2])) >>
-                    dynamicScale;
-
-            accu1 = (lowBandReal[LPC_ORDER + i] >> descale) + (accu1 << 1);
-            accu2 = (lowBandImag[LPC_ORDER + i] >> descale) + (accu2 << 1);
+            accu1 = ((fMultDiv2(a0r, lowBandReal[LPC_ORDER + i - 1]) -
+                      fMultDiv2(a0i, lowBandImag[LPC_ORDER + i - 1])) >>
+                     1) +
+                    ((fMultDiv2(a1r, lowBandReal[LPC_ORDER + i - 2]) -
+                      fMultDiv2(a1i, lowBandImag[LPC_ORDER + i - 2])) >>
+                     1);
+            accu2 = ((fMultDiv2(a0i, lowBandReal[LPC_ORDER + i - 1]) +
+                      fMultDiv2(a0r, lowBandImag[LPC_ORDER + i - 1])) >>
+                     1) +
+                    ((fMultDiv2(a1i, lowBandReal[LPC_ORDER + i - 2]) +
+                      fMultDiv2(a1r, lowBandImag[LPC_ORDER + i - 2])) >>
+                     1);
+
+            accu1 =
+                (lowBandReal[LPC_ORDER + i] >> descale) + (accu1 >> dynscale);
+            accu2 =
+                (lowBandImag[LPC_ORDER + i] >> descale) + (accu2 >> dynscale);
             if (fPreWhitening) {
-              accu1 = scaleValueSaturate(
+              qmfBufferReal[i][hiBand] = scaleValueSaturate(
                   fMultDiv2(accu1, preWhiteningGains[loBand]),
-                  preWhiteningGains_exp[loBand] + 1);
-              accu2 = scaleValueSaturate(
+                  preWhiteningGains_exp[loBand] + 1 + rescale);
+              qmfBufferImag[i][hiBand] = scaleValueSaturate(
                   fMultDiv2(accu2, preWhiteningGains[loBand]),
-                  preWhiteningGains_exp[loBand] + 1);
+                  preWhiteningGains_exp[loBand] + 1 + rescale);
+            } else {
+              qmfBufferReal[i][hiBand] =
+                  SATURATE_LEFT_SHIFT(accu1, rescale, DFRACT_BITS);
+              qmfBufferImag[i][hiBand] =
+                  SATURATE_LEFT_SHIFT(accu2, rescale, DFRACT_BITS);
             }
-            qmfBufferReal[i][hiBand] = accu1;
-            qmfBufferImag[i][hiBand] = accu2;
           }
-#endif
         } else {
           FDK_ASSERT(dynamicScale >= 0);
           calc_qmfBufferReal(