commit 65745694963cc601a2403d071311261f200d6cb9
author Martin Storsjo <martin@martin.st> Fri Aug 10 10:57:24 2012 +0300
committer Martin Storsjo <martin@martin.st> Fri Aug 10 11:05:56 2012 +0300
tree 43d650bf77368c1769881c4bcb65fdf04e51efab
parent d2854e2ac7e7e3cbe4b3c67a2d7823a6153c2488

Don't try to shift more bits than the variable length

Shifting by more than (or equal to) the variable length is
undefined in C.

This caused the quantized spectrum values to go out of range,
triggering asserts later.

Change-Id: If81b6c8caa7b9c75941ad9d280b686d2069c968c

libAACenc/src/quantize.cpp

diff --git a/libAACenc/src/quantize.cpp b/libAACenc/src/quantize.cpp
index 9694901..dc85a6d 100644
--- a/libAACenc/src/quantize.cpp
+++ b/libAACenc/src/quantize.cpp
@@ -127,7 +127,10 @@
       accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
       totalShift = (16-4)-(3*(totalShift>>2));
       FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
-      accu>>=totalShift;
+      if (totalShift < 32)
+          accu>>=totalShift;
+      else
+          accu = 0;
       quaSpectrum[line] = (SHORT)(-((LONG)(k + accu) >> (DFRACT_BITS-1-16)));
     }
     else if(accu > FL2FXCONST_DBL(0.0f))
@@ -140,7 +143,10 @@
       accu = fMultDiv2(FDKaacEnc_mTab_3_4[tabIndex],FDKaacEnc_quantTableE[totalShift&3]);
       totalShift = (16-4)-(3*(totalShift>>2));
       FDK_ASSERT(totalShift >=0); /* MAX_QUANT_VIOLATION */
-      accu>>=totalShift;
+      if (totalShift < 32)
+          accu>>=totalShift;
+      else
+          accu = 0;
       quaSpectrum[line] = (SHORT)((LONG)(k + accu) >> (DFRACT_BITS-1-16));
     }
     else