MediaPlayer: fix channel mask handling

The AAC decoder is publishing the decoded channel mask using the
expected format for KEY_CHANNEL_MASK, which uses the Java AudioFormat
mask definitions.
The MediaPlayer implementation was treating published channel masks
(when found in the decoded format) as using the native mask format,
causing the "stereo Java mask" value of 0b1100 to be handled
as AUDIO_CHANNEL_OUT_FRONT_CENTER | AUDIO_CHANNEL_OUT_LOW_FREQUENCY.
The fix consists in converting from the AudioFormat format to
the audio_channel_mask_t format, unless the two least significant
bits are non-zero (for compatibility with legacy extractors).

Bug: 238115866
Test: repro in bug playing RTSP stream w/ captions on
Change-Id: Ie8b57e20927d90e45b0f69a23a6777af7a39794d
Merged-In: Ie8b57e20927d90e45b0f69a23a6777af7a39794d
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
index cb050f9..45fac76 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
@@ -1946,12 +1946,27 @@
     int32_t numChannels;
     CHECK(format->findInt32("channel-count", &numChannels));
 
-    int32_t rawChannelMask;
-    audio_channel_mask_t channelMask =
-            format->findInt32("channel-mask", &rawChannelMask) ?
-                    static_cast<audio_channel_mask_t>(rawChannelMask)
-                    // signal to the AudioSink to derive the mask from count.
-                    : CHANNEL_MASK_USE_CHANNEL_ORDER;
+    // channel mask info as read from the audio format
+    int32_t channelMaskFromFormat;
+    // channel mask to use for native playback
+    audio_channel_mask_t channelMask;
+    if (format->findInt32("channel-mask", &channelMaskFromFormat)) {
+        // KEY_CHANNEL_MASK follows the android.media.AudioFormat java mask
+        // which is left-bitshifted by 2 relative to the native mask
+        if ((channelMaskFromFormat & 0b11) != 0) {
+            // received an unexpected mask (supposed to follow AudioFormat constants
+            // for output masks with the 2 least-significant bits at 0), but
+            // it may come from an extractor that uses native masks: keeping
+            // the mask as given is ok as it contains at least mono or stereo
+            // and potentially the haptic channels
+            channelMask = static_cast<audio_channel_mask_t>(channelMaskFromFormat);
+        } else {
+            channelMask = static_cast<audio_channel_mask_t>(channelMaskFromFormat >> 2);
+        }
+    } else {
+        // no mask found: the mask will be derived from the channel count
+        channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
+    }
 
     int32_t sampleRate;
     CHECK(format->findInt32("sample-rate", &sampleRate));