hwc: Fix to avoid heap corruption

- Check for MAX_NUM_APP_LAYERS before updating yuv indices array.
- Fall back to GPU composition when number of app layers exceeds
  MAX_NUM_APP_LAYERS to avoid heap corruption.

Bug: 24163261

Acked-by: Ramkumar Radhakrishnan <ramkumar@codeaurora.org>

Change-Id: Ieb91b705a0a5f50ce2f8829d1f1ee048d44b7d2e

msm8960/libhwcomposer/hwc_copybit.cpp

diff --git a/msm8960/libhwcomposer/hwc_copybit.cpp b/msm8960/libhwcomposer/hwc_copybit.cpp
index 98126bf..0839122 100644
--- a/msm8960/libhwcomposer/hwc_copybit.cpp
+++ b/msm8960/libhwcomposer/hwc_copybit.cpp
@@ -159,6 +159,11 @@
         return false;
     }
 
+    if (ctx->listStats[dpy].numAppLayers > MAX_NUM_APP_LAYERS) {
+        // Reached max layers supported by HWC.
+        return false;
+    }
+
     bool useCopybitForYUV = canUseCopybitForYUV(ctx);
     bool useCopybitForRGB = canUseCopybitForRGB(ctx, list, dpy);
     LayerProp *layerProp = ctx->layerProp[dpy];

msm8960/libhwcomposer/hwc_mdpcomp.cpp

diff --git a/msm8960/libhwcomposer/hwc_mdpcomp.cpp b/msm8960/libhwcomposer/hwc_mdpcomp.cpp
index ddf56db..38d8439 100644
--- a/msm8960/libhwcomposer/hwc_mdpcomp.cpp
+++ b/msm8960/libhwcomposer/hwc_mdpcomp.cpp
@@ -346,6 +346,7 @@
 bool MDPComp::isFrameDoable(hwc_context_t *ctx) {
     int numAppLayers = ctx->listStats[mDpy].numAppLayers;
     bool ret = true;
+    const int numAppLayers = ctx->listStats[mDpy].numAppLayers;
 
     if(!isEnabled()) {
         ALOGD_IF(isDebug(),"%s: MDP Comp. not enabled.", __FUNCTION__);
@@ -357,6 +358,9 @@
     } else if(ctx->mVideoTransFlag) {
         ALOGD_IF(isDebug(), "%s: MDP Comp. video transition padding round",
                 __FUNCTION__);
+    } else if(numAppLayers > MAX_NUM_APP_LAYERS) {
+        ALOGD_IF(isDebug(), "%s: Number of App layers exceeded the limit ",
+                 __FUNCTION__);
         ret = false;
     }
     return ret;

msm8960/libhwcomposer/hwc_mdpcomp.h

diff --git a/msm8960/libhwcomposer/hwc_mdpcomp.h b/msm8960/libhwcomposer/hwc_mdpcomp.h
index a0255b7..e2800d2 100644
--- a/msm8960/libhwcomposer/hwc_mdpcomp.h
+++ b/msm8960/libhwcomposer/hwc_mdpcomp.h
@@ -78,7 +78,7 @@
     struct FrameInfo {
         /* maps layer list to mdp list */
         int layerCount;
-        int layerToMDP[MAX_NUM_LAYERS];
+        int layerToMDP[MAX_NUM_APP_LAYERS];
 
         /* maps mdp list to layer list */
         int mdpCount;
@@ -86,7 +86,7 @@
 
         /* layer composing on FB? */
         int fbCount;
-        bool isFBComposed[MAX_NUM_LAYERS];
+        bool isFBComposed[MAX_NUM_APP_LAYERS];
 
         bool needsRedraw;
         int fbZ;
@@ -104,7 +104,7 @@
         int mdpCount;
         int cacheCount;
         int fbZ;
-        buffer_handle_t hnd[MAX_NUM_LAYERS];
+        buffer_handle_t hnd[MAX_NUM_APP_LAYERS];
 
         /* c'tor */
         LayerCache();

msm8960/libhwcomposer/hwc_utils.cpp

diff --git a/msm8960/libhwcomposer/hwc_utils.cpp b/msm8960/libhwcomposer/hwc_utils.cpp
index ce3f281..c68b77a 100644
--- a/msm8960/libhwcomposer/hwc_utils.cpp
+++ b/msm8960/libhwcomposer/hwc_utils.cpp
@@ -431,12 +431,16 @@
     ctx->listStats[dpy].planeAlpha = false;
     ctx->listStats[dpy].yuvCount = 0;
 
-    for (size_t i = 0; i < list->numHwLayers; i++) {
+    //reset yuv indices
+    memset(ctx->listStats[dpy].yuvIndices, -1, MAX_NUM_APP_LAYERS);
+
+    for (size_t i = 0; i < (list->numHwLayers - 1); i++) {
         hwc_layer_1_t const* layer = &list->hwLayers[i];
         private_handle_t *hnd = (private_handle_t *)layer->handle;
 
-        //reset stored yuv index
-        ctx->listStats[dpy].yuvIndices[i] = -1;
+        // continue if i reaches MAX_NUM_APP_LAYERS
+        if(i >= MAX_NUM_APP_LAYERS)
+            continue;
 
         if(list->hwLayers[i].compositionType == HWC_FRAMEBUFFER_TARGET) {
             continue;
@@ -649,8 +653,7 @@
 int hwc_sync(hwc_context_t *ctx, hwc_display_contents_1_t* list, int dpy,
         int fd) {
     int ret = 0;
-
-    int acquireFd[MAX_NUM_LAYERS];
+    int acquireFd[MAX_NUM_APP_LAYERS];
     int count = 0;
     int releaseFd = -1;
     int retireFd = -1;

msm8960/libhwcomposer/hwc_utils.h

diff --git a/msm8960/libhwcomposer/hwc_utils.h b/msm8960/libhwcomposer/hwc_utils.h
index ee1cbe8..aa66f90 100644
--- a/msm8960/libhwcomposer/hwc_utils.h
+++ b/msm8960/libhwcomposer/hwc_utils.h
@@ -35,7 +35,7 @@
 #define ALIGN_TO(x, align)     (((x) + ((align)-1)) & ~((align)-1))
 #define LIKELY( exp )       (__builtin_expect( (exp) != 0, true  ))
 #define UNLIKELY( exp )     (__builtin_expect( (exp) != 0, false ))
-#define MAX_NUM_LAYERS 32 //includes fb layer
+#define MAX_NUM_APP_LAYERS 32
 #define MAX_DISPLAY_DIM 2048
 
 // For support of virtual displays
@@ -91,7 +91,7 @@
     int fbLayerIndex; //Always last for now. = numAppLayers
     //Video specific
     int yuvCount;
-    int yuvIndices[MAX_NUM_LAYERS];
+    int yuvIndices[MAX_NUM_APP_LAYERS];
     bool needsAlphaScale;
     bool preMultipliedAlpha;
     bool planeAlpha;