cas: do not use hidl_memory if size is > SIZE_MAX

Bug: 79376389
Test:
POC in bug; CTS MediaCasTest.

Change-Id: Ia3ecf9664ab69ce293e10b73bc20c37a9992db9f
(cherry picked from commit e1ce74509e06f56fffa484d76aada8c429397b26)
diff --git a/cas/1.0/default/DescramblerImpl.cpp b/cas/1.0/default/DescramblerImpl.cpp
index 1f89933..6d5e2d5 100644
--- a/cas/1.0/default/DescramblerImpl.cpp
+++ b/cas/1.0/default/DescramblerImpl.cpp
@@ -96,13 +96,13 @@
         descramble_cb _hidl_cb) {
     ALOGV("%s", __FUNCTION__);
 
-    // Get a local copy of the shared_ptr for the plugin. Note that before
-    // calling the HIDL callback, this shared_ptr must be manually reset,
-    // since the client side could proceed as soon as the callback is called
-    // without waiting for this method to go out of scope.
-    std::shared_ptr<DescramblerPlugin> holder = std::atomic_load(&mPluginHolder);
-    if (holder.get() == nullptr) {
-        _hidl_cb(toStatus(INVALID_OPERATION), 0, NULL);
+    // hidl_memory's size is stored in uint64_t, but mapMemory's mmap will map
+    // size in size_t. If size is over SIZE_MAX, mapMemory mapMemory could succeed
+    // but the mapped memory's actual size will be smaller than the reported size.
+    if (srcBuffer.heapBase.size() > SIZE_MAX) {
+        ALOGE("Invalid hidl_memory size: %llu", srcBuffer.heapBase.size());
+        android_errorWriteLog(0x534e4554, "79376389");
+        _hidl_cb(toStatus(BAD_VALUE), 0, NULL);
         return Void();
     }
 
@@ -112,7 +112,6 @@
     // mapped ashmem, since the offset and size is controlled by client.
     if (srcMem == NULL) {
         ALOGE("Failed to map src buffer.");
-        holder.reset();
         _hidl_cb(toStatus(BAD_VALUE), 0, NULL);
         return Void();
     }
@@ -121,7 +120,6 @@
         ALOGE("Invalid src buffer range: offset %llu, size %llu, srcMem size %llu",
                 srcBuffer.offset, srcBuffer.size, (uint64_t)srcMem->getSize());
         android_errorWriteLog(0x534e4554, "67962232");
-        holder.reset();
         _hidl_cb(toStatus(BAD_VALUE), 0, NULL);
         return Void();
     }
@@ -139,7 +137,6 @@
                 "srcOffset %llu, totalBytesInSubSamples %llu, srcBuffer size %llu",
                 srcOffset, totalBytesInSubSamples, srcBuffer.size);
         android_errorWriteLog(0x534e4554, "67962232");
-        holder.reset();
         _hidl_cb(toStatus(BAD_VALUE), 0, NULL);
         return Void();
     }
@@ -158,7 +155,6 @@
                     "dstOffset %llu, totalBytesInSubSamples %llu, srcBuffer size %llu",
                     dstOffset, totalBytesInSubSamples, srcBuffer.size);
             android_errorWriteLog(0x534e4554, "67962232");
-            holder.reset();
             _hidl_cb(toStatus(BAD_VALUE), 0, NULL);
             return Void();
         }
@@ -167,6 +163,17 @@
                 dstBuffer.secureMemory.getNativeHandle());
         dstPtr = static_cast<void *>(handle);
     }
+
+    // Get a local copy of the shared_ptr for the plugin. Note that before
+    // calling the HIDL callback, this shared_ptr must be manually reset,
+    // since the client side could proceed as soon as the callback is called
+    // without waiting for this method to go out of scope.
+    std::shared_ptr<DescramblerPlugin> holder = std::atomic_load(&mPluginHolder);
+    if (holder.get() == nullptr) {
+        _hidl_cb(toStatus(INVALID_OPERATION), 0, NULL);
+        return Void();
+    }
+
     // Casting hidl SubSample to DescramblerPlugin::SubSample, but need
     // to ensure structs are actually idential