commit 55002323b3a94a8a827b66acc693cbdd38e57119
author Fraunhofer IIS FDK <audio-fdk@iis.fraunhofer.de> Fri Mar 08 16:08:24 2019 +0100
committer Jean-Michel Trivi <jmtrivi@google.com> Wed May 15 12:31:19 2019 -0700
tree 89d4fc72c67bf3d849f4a4b2f6d38c6167f09749
parent f6a6a071a9b39fd62d0f81ddba793c00cae68a49

Fix integer overflow in FDKaacEnc_IntensityStereoProcessing().

Bug: 132641988
Test: atest DecoderTestXheAac ; atest DecoderTestAacDrc
Change-Id: Id2a7e26fff499146a39cea757b1b7fc787b54b31

libAACenc/src/intensity.cpp

diff --git a/libAACenc/src/intensity.cpp b/libAACenc/src/intensity.cpp
index a160a4f..0944fa3 100644
--- a/libAACenc/src/intensity.cpp
+++ b/libAACenc/src/intensity.cpp
@@ -1,7 +1,7 @@
 /* -----------------------------------------------------------------------------
 Software License for The Fraunhofer FDK AAC Codec Library for Android
 
-© Copyright  1995 - 2018 Fraunhofer-Gesellschaft zur Förderung der angewandten
+© Copyright  1995 - 2019 Fraunhofer-Gesellschaft zur Förderung der angewandten
 Forschung e.V. All rights reserved.
 
  1.    INTRODUCTION
@@ -661,6 +661,7 @@
     for (sfboffs = 0; sfboffs < maxSfbPerGroup; sfboffs++) {
       INT sL, sR;
       FIXP_DBL inv_n;
+      INT mdct_spec_sf = MDCT_SPEC_SF;
 
       msMask[sfb + sfboffs] = 0;
       if (isMask[sfb + sfboffs] == 0) {
@@ -682,6 +683,12 @@
         }
       }
 
+      if (sfbOffset[sfb + sfboffs + 1] - sfbOffset[sfb + sfboffs] >
+          1 << mdct_spec_sf) {
+        mdct_spec_sf++; /* This is for rare cases where the number of bins in a
+                           scale factor band is > 64 */
+      }
+
       inv_n = GetInvInt(
           (sfbOffset[sfb + sfboffs + 1] - sfbOffset[sfb + sfboffs]) >>
           1);  // scaled with 2 to compensate fMultDiv2() in subsequent loop
@@ -707,11 +714,11 @@
              j++) {
           d = ((mdctSpectrumLeft[j] << s0) >> 1) -
               ((mdctSpectrumRight[j] << s0) >> 1);
-          ed += fMultDiv2(d, d) >> (MDCT_SPEC_SF - 1);
+          ed += fMultDiv2(d, d) >> (mdct_spec_sf - 1);
         }
         msMask[sfb + sfboffs] = 1;
         tmp = fDivNorm(sfbEnergyLeft[sfb + sfboffs], ed, &s1);
-        s2 = (s1) + (2 * s0) - 2 - MDCT_SPEC_SF;
+        s2 = (s1) + (2 * s0) - 2 - mdct_spec_sf;
         if (s2 & 1) {
           tmp = tmp >> 1;
           s2 = s2 + 1;
@@ -748,12 +755,12 @@
           s = ((mdctSpectrumLeft[j] << s0) >> 1) +
               ((mdctSpectrumRight[j] << s0) >> 1);
           es += fMultDiv2(s, s) >>
-                (MDCT_SPEC_SF -
-                 1);  // scaled 2*(mdctScale - s0 + 1) + MDCT_SPEC_SF
+                (mdct_spec_sf -
+                 1);  // scaled 2*(mdctScale - s0 + 1) + mdct_spec_sf
         }
         msMask[sfb + sfboffs] = 0;
         tmp = fDivNorm(sfbEnergyLeft[sfb + sfboffs], es, &s1);
-        s2 = (s1) + (2 * s0) - 2 - MDCT_SPEC_SF;
+        s2 = (s1) + (2 * s0) - 2 - mdct_spec_sf;
         if (s2 & 1) {
           tmp = tmp >> 1;
           s2 = s2 + 1;