Avoid decoder internal clipping by converting the whole audio sample data path from 16 to 32 bit data width (FDKdec v3.2.0).

Bug: 149514474
Test: atest DecoderTestXheAac DecoderTestAacDrc
Change-Id: I8a504ab709e42e27a61fe29840212953742283a5

1 files changed, 19 insertions, 16 deletions

diff --git a/libAACdec/src/ldfiltbank.cpp b/libAACdec/src/ldfiltbank.cpp
index 1898557..13e61a5 100644
--- a/libAACdec/src/ldfiltbank.cpp
+++ b/libAACdec/src/ldfiltbank.cpp
@@ -112,17 +112,20 @@ amm-info@iis.fraunhofer.de
 #if defined(__arm__)
 #endif
 
-static void multE2_DinvF_fdk(FIXP_PCM *output, FIXP_DBL *x, const FIXP_WTB *fb,
+static void multE2_DinvF_fdk(PCM_DEC *output, FIXP_DBL *x, const FIXP_WTB *fb,
                              FIXP_DBL *z, const int N) {
   int i;
 
-  /*  scale for FIXP_DBL -> INT_PCM conversion. */
-  const int scale = (DFRACT_BITS - SAMPLE_BITS) - LDFB_HEADROOM;
-#if ((DFRACT_BITS - SAMPLE_BITS - LDFB_HEADROOM) > 0)
+  /*  scale for FIXP_DBL -> PCM_DEC conversion:       */
+  const int scale = (DFRACT_BITS - PCM_OUT_BITS) - LDFB_HEADROOM + (3);
+
+#if ((DFRACT_BITS - PCM_OUT_BITS - LDFB_HEADROOM + (3) - 1) > 0)
   FIXP_DBL rnd_val_wts0 = (FIXP_DBL)0;
   FIXP_DBL rnd_val_wts1 = (FIXP_DBL)0;
+#if ((DFRACT_BITS - PCM_OUT_BITS - LDFB_HEADROOM + (3) - WTS0 - 1) > 0)
   if (-WTS0 - 1 + scale)
     rnd_val_wts0 = (FIXP_DBL)(1 << (-WTS0 - 1 + scale - 1));
+#endif
   if (-WTS1 - 1 + scale)
     rnd_val_wts1 = (FIXP_DBL)(1 << (-WTS1 - 1 + scale - 1));
 #endif
@@ -141,16 +144,16 @@ static void multE2_DinvF_fdk(FIXP_PCM *output, FIXP_DBL *x, const FIXP_WTB *fb,
     tmp = (fMultDiv2(z[N / 2 + i], fb[N + N / 2 - 1 - i]) +
            fMultDiv2(z[i], fb[N + N / 2 + i]));
 
-#if ((DFRACT_BITS - SAMPLE_BITS - LDFB_HEADROOM) > 0)
+#if ((DFRACT_BITS - PCM_OUT_BITS - LDFB_HEADROOM + (3) - 1) > 0)
     FDK_ASSERT((-WTS1 - 1 + scale) >= 0);
     FDK_ASSERT(tmp <= ((FIXP_DBL)0x7FFFFFFF -
                        rnd_val_wts1)); /* rounding must not cause overflow */
-    output[(N * 3 / 4 - 1 - i)] = (FIXP_PCM)SATURATE_RIGHT_SHIFT(
+    output[(N * 3 / 4 - 1 - i)] = (PCM_DEC)SATURATE_RIGHT_SHIFT(
         tmp + rnd_val_wts1, -WTS1 - 1 + scale, PCM_OUT_BITS);
 #else
     FDK_ASSERT((WTS1 + 1 - scale) >= 0);
     output[(N * 3 / 4 - 1 - i)] =
-        (FIXP_PCM)SATURATE_LEFT_SHIFT(tmp, WTS1 + 1 - scale, PCM_OUT_BITS);
+        (PCM_DEC)SATURATE_LEFT_SHIFT(tmp, WTS1 + 1 - scale, PCM_OUT_BITS);
 #endif
 
     z[i] = z0;
@@ -173,22 +176,22 @@ static void multE2_DinvF_fdk(FIXP_PCM *output, FIXP_DBL *x, const FIXP_WTB *fb,
     tmp1 = (fMultDiv2(z[N / 2 + i], fb[N + N / 2 - 1 - i]) +
             fMultDiv2(z[i], fb[N + N / 2 + i]));
 
-#if ((DFRACT_BITS - SAMPLE_BITS - LDFB_HEADROOM) > 0)
+#if ((DFRACT_BITS - PCM_OUT_BITS - LDFB_HEADROOM + (3) - 1) > 0)
     FDK_ASSERT((-WTS0 - 1 + scale) >= 0);
     FDK_ASSERT(tmp0 <= ((FIXP_DBL)0x7FFFFFFF -
                         rnd_val_wts0)); /* rounding must not cause overflow */
     FDK_ASSERT(tmp1 <= ((FIXP_DBL)0x7FFFFFFF -
                         rnd_val_wts1)); /* rounding must not cause overflow */
-    output[(i - N / 4)] = (FIXP_PCM)SATURATE_RIGHT_SHIFT(
+    output[(i - N / 4)] = (PCM_DEC)SATURATE_RIGHT_SHIFT(
         tmp0 + rnd_val_wts0, -WTS0 - 1 + scale, PCM_OUT_BITS);
-    output[(N * 3 / 4 - 1 - i)] = (FIXP_PCM)SATURATE_RIGHT_SHIFT(
+    output[(N * 3 / 4 - 1 - i)] = (PCM_DEC)SATURATE_RIGHT_SHIFT(
         tmp1 + rnd_val_wts1, -WTS1 - 1 + scale, PCM_OUT_BITS);
 #else
     FDK_ASSERT((WTS0 + 1 - scale) >= 0);
     output[(i - N / 4)] =
-        (FIXP_PCM)SATURATE_LEFT_SHIFT(tmp0, WTS0 + 1 - scale, PCM_OUT_BITS);
+        (PCM_DEC)SATURATE_LEFT_SHIFT(tmp0, WTS0 + 1 - scale, PCM_OUT_BITS);
     output[(N * 3 / 4 - 1 - i)] =
-        (FIXP_PCM)SATURATE_LEFT_SHIFT(tmp1, WTS1 + 1 - scale, PCM_OUT_BITS);
+        (PCM_DEC)SATURATE_LEFT_SHIFT(tmp1, WTS1 + 1 - scale, PCM_OUT_BITS);
 #endif
     z[i] = z0;
     z[N + i] = z2;
@@ -198,22 +201,22 @@ static void multE2_DinvF_fdk(FIXP_PCM *output, FIXP_DBL *x, const FIXP_WTB *fb,
   for (i = 0; i < N / 4; i++) {
     FIXP_DBL tmp0 = fMultDiv2(z[i], fb[N / 2 + i]);
 
-#if ((DFRACT_BITS - SAMPLE_BITS - LDFB_HEADROOM) > 0)
+#if ((DFRACT_BITS - PCM_OUT_BITS - LDFB_HEADROOM + (3) - 1) > 0)
     FDK_ASSERT((-WTS0 - 1 + scale) >= 0);
     FDK_ASSERT(tmp0 <= ((FIXP_DBL)0x7FFFFFFF -
                         rnd_val_wts0)); /* rounding must not cause overflow */
-    output[(N * 3 / 4 + i)] = (FIXP_PCM)SATURATE_RIGHT_SHIFT(
+    output[(N * 3 / 4 + i)] = (PCM_DEC)SATURATE_RIGHT_SHIFT(
         tmp0 + rnd_val_wts0, -WTS0 - 1 + scale, PCM_OUT_BITS);
 #else
     FDK_ASSERT((WTS0 + 1 - scale) >= 0);
     output[(N * 3 / 4 + i)] =
-        (FIXP_PCM)SATURATE_LEFT_SHIFT(tmp0, WTS0 + 1 - scale, PCM_OUT_BITS);
+        (PCM_DEC)SATURATE_LEFT_SHIFT(tmp0, WTS0 + 1 - scale, PCM_OUT_BITS);
 #endif
   }
 }
 
 int InvMdctTransformLowDelay_fdk(FIXP_DBL *mdctData, const int mdctData_e,
-                                 FIXP_PCM *output, FIXP_DBL *fs_buffer,
+                                 PCM_DEC *output, FIXP_DBL *fs_buffer,
                                  const int N) {
   const FIXP_WTB *coef;
   FIXP_DBL gain = (FIXP_DBL)0;