Merge "Check integer overflow to prevent memory corruption" into klp-dev
diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h
index bf3a998..0783e2b 100644
--- a/include/media/stagefright/ACodec.h
+++ b/include/media/stagefright/ACodec.h
@@ -188,6 +188,7 @@
bool mSentFormat;
bool mIsEncoder;
bool mUseMetadataOnEncoderOutput;
+ bool mFatalError;
bool mShutdownInProgress;
bool mIsConfiguredForAdaptivePlayback;
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 9276818..9fda07f 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -362,6 +362,7 @@
mSentFormat(false),
mIsEncoder(false),
mUseMetadataOnEncoderOutput(false),
+ mFatalError(false),
mShutdownInProgress(false),
mIsConfiguredForAdaptivePlayback(false),
mEncoderDelay(0),
@@ -812,6 +813,12 @@
ANativeWindowBuffer *buf;
int fenceFd = -1;
CHECK(mNativeWindow.get() != NULL);
+
+ if (mFatalError) {
+ ALOGW("not dequeuing from native window due to fatal error");
+ return NULL;
+ }
+
if (native_window_dequeue_buffer_and_wait(mNativeWindow.get(), &buf) != 0) {
ALOGE("dequeueBuffer failed.");
return NULL;
@@ -2692,6 +2699,9 @@
sp<AMessage> notify = mNotify->dup();
notify->setInt32("what", ACodec::kWhatError);
notify->setInt32("omx-error", error);
+
+ mFatalError = true;
+
notify->setInt32("err", internalError);
notify->post();
}
diff --git a/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/MPEG4Extractor.cpp
index 56bd875..0899362 100644
--- a/media/libstagefright/MPEG4Extractor.cpp
+++ b/media/libstagefright/MPEG4Extractor.cpp
@@ -1772,7 +1772,7 @@
ALOGV("chunk_data_size = %lld and data_offset = %lld",
chunk_data_size, data_offset);
- if (chunk_data_size >= SIZE_MAX - 1) {
+ if (chunk_data_size < 0 || static_cast<uint64_t>(chunk_data_size) >= SIZE_MAX - 1) {
return ERROR_MALFORMED;
}
sp<ABuffer> buffer = new ABuffer(chunk_data_size + 1);
@@ -2747,8 +2747,17 @@
int ivlength;
CHECK(mFormat->findInt32(kKeyCryptoDefaultIVSize, &ivlength));
+ // only 0, 8 and 16 byte initialization vectors are supported
+ if (ivlength != 0 && ivlength != 8 && ivlength != 16) {
+ ALOGW("unsupported IV length: %d", ivlength);
+ return ERROR_MALFORMED;
+ }
// read CencSampleAuxiliaryDataFormats
for (size_t i = 0; i < mCurrentSampleInfoCount; i++) {
+ if (i >= mCurrentSamples.size()) {
+ ALOGW("too few samples");
+ break;
+ }
Sample *smpl = &mCurrentSamples.editItemAt(i);
memset(smpl->iv, 0, 16);
@@ -3193,6 +3202,10 @@
CHECK(mBuffer == NULL);
return err;
}
+ if (size > mBuffer->size()) {
+ ALOGE("buffer too small: %zu > %zu", size, mBuffer->size());
+ return ERROR_BUFFER_TOO_SMALL;
+ }
}
if (!mIsAVC || mWantsNALFragments) {
@@ -3439,6 +3452,10 @@
ALOGV("acquire_buffer returned %d", err);
return err;
}
+ if (size > mBuffer->size()) {
+ ALOGE("buffer too small: %zu > %zu", size, mBuffer->size());
+ return ERROR_BUFFER_TOO_SMALL;
+ }
}
const Sample *smpl = &mCurrentSamples[mCurrentSampleIndex];
@@ -3458,6 +3475,14 @@
if (!mIsAVC || mWantsNALFragments) {
if (newBuffer) {
+ if (!isInRange((size_t)0u, mBuffer->size(), size)) {
+ mBuffer->release();
+ mBuffer = NULL;
+
+ ALOGE("fragmentedRead ERROR_MALFORMED size %zu", size);
+ return ERROR_MALFORMED;
+ }
+
ssize_t num_bytes_read =
mDataSource->readAt(offset, (uint8_t *)mBuffer->data(), size);
@@ -3465,7 +3490,7 @@
mBuffer->release();
mBuffer = NULL;
- ALOGV("i/o error");
+ ALOGE("i/o error");
return ERROR_IO;
}
@@ -3535,18 +3560,40 @@
ssize_t num_bytes_read = 0;
int32_t drm = 0;
bool usesDRM = (mFormat->findInt32(kKeyIsDRM, &drm) && drm != 0);
+ void *data = NULL;
+ bool isMalFormed = false;
if (usesDRM) {
- num_bytes_read =
- mDataSource->readAt(offset, (uint8_t*)mBuffer->data(), size);
+ if (mBuffer == NULL || !isInRange((size_t)0u, mBuffer->size(), size)) {
+ isMalFormed = true;
+ } else {
+ data = mBuffer->data();
+ }
} else {
- num_bytes_read = mDataSource->readAt(offset, mSrcBuffer, size);
+ int32_t max_size;
+ if (mFormat == NULL
+ || !mFormat->findInt32(kKeyMaxInputSize, &max_size)
+ || !isInRange((size_t)0u, (size_t)max_size, size)) {
+ isMalFormed = true;
+ } else {
+ data = mSrcBuffer;
+ }
}
+ if (isMalFormed || data == NULL) {
+ ALOGE("isMalFormed size %zu", size);
+ if (mBuffer != NULL) {
+ mBuffer->release();
+ mBuffer = NULL;
+ }
+ return ERROR_MALFORMED;
+ }
+ num_bytes_read = mDataSource->readAt(offset, data, size);
+
if (num_bytes_read < (ssize_t)size) {
mBuffer->release();
mBuffer = NULL;
- ALOGV("i/o error");
+ ALOGE("i/o error");
return ERROR_IO;
}
@@ -3560,16 +3607,18 @@
size_t dstOffset = 0;
while (srcOffset < size) {
- bool isMalFormed = !isInRange(0u, size, srcOffset, mNALLengthSize);
+ isMalFormed = !isInRange((size_t)0u, size, srcOffset, mNALLengthSize);
size_t nalLength = 0;
if (!isMalFormed) {
nalLength = parseNALSize(&mSrcBuffer[srcOffset]);
srcOffset += mNALLengthSize;
- isMalFormed = !isInRange(0u, size, srcOffset, nalLength);
+ isMalFormed = !isInRange((size_t)0u, size, srcOffset, nalLength)
+ || !isInRange((size_t)0u, mBuffer->size(), dstOffset, (size_t)4u)
+ || !isInRange((size_t)0u, mBuffer->size(), dstOffset + 4, nalLength);
}
if (isMalFormed) {
- ALOGE("Video is malformed");
+ ALOGE("Video is malformed; nalLength %zu", nalLength);
mBuffer->release();
mBuffer = NULL;
return ERROR_MALFORMED;
diff --git a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
index 1d66120..4e11628 100644
--- a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
+++ b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
@@ -34,6 +34,10 @@
#include "SoftAVCEncoder.h"
+#ifndef INT32_MAX
+#define INT32_MAX 2147483647
+#endif
+
namespace android {
template<class T>
@@ -257,6 +261,10 @@
if (mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) {
// Color conversion is needed.
CHECK(mInputFrameData == NULL);
+ if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {
+ ALOGE("Buffer size is too big.");
+ return OMX_ErrorUndefined;
+ }
mInputFrameData =
(uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);
CHECK(mInputFrameData != NULL);
@@ -278,6 +286,10 @@
int32_t nMacroBlocks = ((((mVideoWidth + 15) >> 4) << 4) *
(((mVideoHeight + 15) >> 4) << 4)) >> 8;
CHECK(mSliceGroup == NULL);
+ if ((size_t)nMacroBlocks > SIZE_MAX / sizeof(uint32_t)) {
+ ALOGE("requested memory size is too big.");
+ return OMX_ErrorUndefined;
+ }
mSliceGroup = (uint32_t *) malloc(sizeof(uint32_t) * nMacroBlocks);
CHECK(mSliceGroup != NULL);
for (int ii = 0, idx = 0; ii < nMacroBlocks; ++ii) {
@@ -698,6 +710,10 @@
if (mStoreMetaDataInBuffers) {
mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar;
if (mInputFrameData == NULL) {
+ if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {
+ ALOGE("Buffer size is too big.");
+ return OMX_ErrorUndefined;
+ }
mInputFrameData =
(uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);
}
diff --git a/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp b/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp
index 844bd14..419e3f4 100644
--- a/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp
+++ b/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp
@@ -19,6 +19,14 @@
#include "vlc_decode.h"
#include "bitstream.h"
+#ifndef INT32_MAX
+#define INT32_MAX 0x7fffffff
+#endif
+
+#ifndef SIZE_MAX
+#define SIZE_MAX ((size_t) -1)
+#endif
+
#define OSCL_DISABLE_WARNING_CONDITIONAL_IS_CONSTANT
#ifdef DEC_INTERNAL_MEMORY_OPT
@@ -95,6 +103,11 @@
#ifdef DEC_INTERNAL_MEMORY_OPT
video->vol = (Vol **) IMEM_VOL;
#else
+ if ((size_t)nLayers > SIZE_MAX / sizeof(Vol *)) {
+ status = PV_FALSE;
+ goto fail;
+ }
+
video->vol = (Vol **) oscl_malloc(nLayers * sizeof(Vol *));
#endif
if (video->vol == NULL) status = PV_FALSE;
@@ -128,6 +141,11 @@
else oscl_memset(video->prevVop, 0, sizeof(Vop));
video->memoryUsage += (sizeof(Vop) * 2);
+ if ((size_t)nLayers > SIZE_MAX / sizeof(Vop *)) {
+ status = PV_FALSE;
+ goto fail;
+ }
+
video->vopHeader = (Vop **) oscl_malloc(sizeof(Vop *) * nLayers);
#endif
if (video->vopHeader == NULL) status = PV_FALSE;
@@ -274,6 +292,7 @@
status = PV_FALSE;
}
+fail:
if (status == PV_FALSE) PVCleanUpVideoDecoder(decCtrl);
return status;
@@ -300,6 +319,10 @@
video->nMBPerRow * video->nMBPerCol;
}
+ if (((uint64_t)video->width * video->height) > (uint64_t)INT32_MAX / sizeof(PIXEL)) {
+ return PV_FALSE;
+ }
+
size = (int32)sizeof(PIXEL) * video->width * video->height;
#ifdef PV_MEMORY_POOL
decCtrl->size = size;
@@ -315,6 +338,9 @@
video->prevVop->uChan = video->prevVop->yChan + size;
video->prevVop->vChan = video->prevVop->uChan + (size >> 2);
#else
+ if (size > INT32_MAX / 3 * 2) {
+ return PV_FALSE;
+ }
video->currVop->yChan = (PIXEL *) oscl_malloc(size * 3 / 2); /* Allocate memory for all VOP OKA 3/2/1*/
if (video->currVop->yChan == NULL) status = PV_FALSE;
@@ -342,6 +368,10 @@
{
oscl_memset(video->prevEnhcVop, 0, sizeof(Vop));
#ifndef PV_MEMORY_POOL
+ if (size > INT32_MAX / 3 * 2) {
+ return PV_FALSE;
+ }
+
video->prevEnhcVop->yChan = (PIXEL *) oscl_malloc(size * 3 / 2); /* Allocate memory for all VOP OKA 3/2/1*/
if (video->prevEnhcVop->yChan == NULL) status = PV_FALSE;
video->prevEnhcVop->uChan = video->prevEnhcVop->yChan + size;
@@ -398,10 +428,17 @@
if (video->acPredFlag == NULL) status = PV_FALSE;
video->memoryUsage += (nTotalMB);
+ if ((size_t)nTotalMB > SIZE_MAX / sizeof(typeDCStore)) {
+ return PV_FALSE;
+ }
video->predDC = (typeDCStore *) oscl_malloc(nTotalMB * sizeof(typeDCStore));
if (video->predDC == NULL) status = PV_FALSE;
video->memoryUsage += (nTotalMB * sizeof(typeDCStore));
+ if (nMBPerRow > INT32_MAX - 1
+ || (size_t)(nMBPerRow + 1) > SIZE_MAX / sizeof(typeDCACStore)) {
+ return PV_FALSE;
+ }
video->predDCAC_col = (typeDCACStore *) oscl_malloc((nMBPerRow + 1) * sizeof(typeDCACStore));
if (video->predDCAC_col == NULL) status = PV_FALSE;
video->memoryUsage += ((nMBPerRow + 1) * sizeof(typeDCACStore));
@@ -417,6 +454,10 @@
video->headerInfo.CBP = (uint8 *) oscl_malloc(nTotalMB);
if (video->headerInfo.CBP == NULL) status = PV_FALSE;
video->memoryUsage += nTotalMB;
+
+ if ((size_t)nTotalMB > SIZE_MAX / sizeof(int16)) {
+ return PV_FALSE;
+ }
video->QPMB = (int16 *) oscl_malloc(nTotalMB * sizeof(int16));
if (video->QPMB == NULL) status = PV_FALSE;
video->memoryUsage += (nTotalMB * sizeof(int));
@@ -434,6 +475,9 @@
video->memoryUsage += sizeof(MacroBlock);
}
/* Allocating motion vector space */
+ if ((size_t)nTotalMB > SIZE_MAX / (sizeof(MOT) * 4)) {
+ return PV_FALSE;
+ }
video->motX = (MOT *) oscl_malloc(sizeof(MOT) * 4 * nTotalMB);
if (video->motX == NULL) status = PV_FALSE;
video->motY = (MOT *) oscl_malloc(sizeof(MOT) * 4 * nTotalMB);
@@ -467,6 +511,9 @@
}
#else
+ if (nTotalMB > INT32_MAX / 6) {
+ return PV_FALSE;
+ }
video->pstprcTypCur = (uint8 *) oscl_malloc(nTotalMB * 6);
video->memoryUsage += (nTotalMB * 6);
if (video->pstprcTypCur == NULL)
diff --git a/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp b/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp
index 946e3d0..c2b7c8d 100644
--- a/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp
+++ b/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp
@@ -21,6 +21,13 @@
#include "rate_control.h"
#include "m4venc_oscl.h"
+#ifndef INT32_MAX
+#define INT32_MAX 0x7fffffff
+#endif
+
+#ifndef SIZE_MAX
+#define SIZE_MAX ((size_t) -1)
+#endif
/* Inverse normal zigzag */
const static Int zigzag_i[NCOEFF_BLOCK] =
@@ -610,6 +617,10 @@
max = temp_w * temp_h;
max_width = ((temp_w + 15) >> 4) << 4;
max_height = ((temp_h + 15) >> 4) << 4;
+ if (((uint64_t)max_width * max_height) > (uint64_t)INT32_MAX
+ || temp_w > INT32_MAX - 15 || temp_h > INT32_MAX - 15) {
+ goto CLEAN_UP;
+ }
nTotalMB = ((max_width * max_height) >> 8);
}
@@ -654,6 +665,9 @@
/* Allocating motion vector space and interpolation memory*/
+ if ((size_t)nTotalMB > SIZE_MAX / sizeof(MOT *)) {
+ goto CLEAN_UP;
+ }
video->mot = (MOT **)M4VENC_MALLOC(sizeof(MOT *) * nTotalMB);
if (video->mot == NULL) goto CLEAN_UP;
@@ -676,11 +690,17 @@
/* so that compilers can generate faster code to indexing the */
/* data inside (by using << instead of *). 04/14/2000. */
/* 5/29/01, use decoder lib ACDC prediction memory scheme. */
+ if ((size_t)nTotalMB > SIZE_MAX / sizeof(typeDCStore)) {
+ goto CLEAN_UP;
+ }
video->predDC = (typeDCStore *) M4VENC_MALLOC(nTotalMB * sizeof(typeDCStore));
if (video->predDC == NULL) goto CLEAN_UP;
if (!video->encParams->H263_Enabled)
{
+ if ((size_t)((max_width >> 4) + 1) > SIZE_MAX / sizeof(typeDCACStore)) {
+ goto CLEAN_UP;
+ }
video->predDCAC_col = (typeDCACStore *) M4VENC_MALLOC(((max_width >> 4) + 1) * sizeof(typeDCACStore));
if (video->predDCAC_col == NULL) goto CLEAN_UP;
@@ -688,6 +708,9 @@
/* the rest will be used for storing horizontal (row) AC coefficients */
video->predDCAC_row = video->predDCAC_col + 1; /* ACDC */
+ if ((size_t)nTotalMB > SIZE_MAX / sizeof(Int)) {
+ goto CLEAN_UP;
+ }
video->acPredFlag = (Int *) M4VENC_MALLOC(nTotalMB * sizeof(Int)); /* Memory for acPredFlag */
if (video->acPredFlag == NULL) goto CLEAN_UP;
}
@@ -741,8 +764,15 @@
offset = (pitch << 4) + 16;
max_height += 32;
}
+ if (((uint64_t)pitch * max_height) > (uint64_t)INT32_MAX) {
+ goto CLEAN_UP;
+ }
size = pitch * max_height;
+ if (size > INT32_MAX - (size >> 1)
+ || (size_t)(size + (size >> 1)) > SIZE_MAX / sizeof(PIXEL)) {
+ goto CLEAN_UP;
+ }
video->currVop->yChan = (PIXEL *)M4VENC_MALLOC(sizeof(PIXEL) * (size + (size >> 1))); /* Memory for currVop Y */
if (video->currVop->yChan == NULL) goto CLEAN_UP;
video->currVop->uChan = video->currVop->yChan + size;/* Memory for currVop U */
@@ -841,6 +871,9 @@
/* /// End /////////////////////////////////////// */
+ if ((size_t)nLayers > SIZE_MAX / sizeof(Vol *)) {
+ goto CLEAN_UP;
+ }
video->vol = (Vol **)M4VENC_MALLOC(nLayers * sizeof(Vol *)); /* Memory for VOL pointers */
/* Memory allocation and Initialization of Vols and writing of headers */
diff --git a/media/libstagefright/foundation/ABuffer.cpp b/media/libstagefright/foundation/ABuffer.cpp
index 6173db4..5f5c977 100644
--- a/media/libstagefright/foundation/ABuffer.cpp
+++ b/media/libstagefright/foundation/ABuffer.cpp
@@ -23,12 +23,17 @@
namespace android {
ABuffer::ABuffer(size_t capacity)
- : mData(malloc(capacity)),
- mCapacity(capacity),
- mRangeOffset(0),
- mRangeLength(capacity),
+ : mRangeOffset(0),
mInt32Data(0),
mOwnsData(true) {
+ mData = malloc(capacity);
+ if (mData == NULL) {
+ mCapacity = 0;
+ mRangeLength = 0;
+ } else {
+ mCapacity = capacity;
+ mRangeLength = capacity;
+ }
}
ABuffer::ABuffer(void *data, size_t capacity)
diff --git a/media/libstagefright/id3/ID3.cpp b/media/libstagefright/id3/ID3.cpp
index d34f1a7..751b810 100644
--- a/media/libstagefright/id3/ID3.cpp
+++ b/media/libstagefright/id3/ID3.cpp
@@ -349,7 +349,7 @@
if (flags & 1) {
// Strip data length indicator
- if (mSize < 14 || mSize - 14 < offset) {
+ if (mSize < 14 || mSize - 14 < offset || dataSize < 4) {
return false;
}
memmove(&mData[offset + 10], &mData[offset + 14], mSize - offset - 14);
@@ -550,6 +550,9 @@
return;
}
+ if (mFrameSize < getHeaderLength() + 1) {
+ return;
+ }
size_t n = mFrameSize - getHeaderLength() - 1;
if (otherdata) {
// skip past the encoding, language, and the 0 separator
diff --git a/media/libstagefright/matroska/MatroskaExtractor.cpp b/media/libstagefright/matroska/MatroskaExtractor.cpp
index cf20428..25d0cf1 100644
--- a/media/libstagefright/matroska/MatroskaExtractor.cpp
+++ b/media/libstagefright/matroska/MatroskaExtractor.cpp
@@ -23,6 +23,7 @@
#include "mkvparser.hpp"
#include <media/stagefright/foundation/ADebug.h>
+#include <media/stagefright/foundation/AUtils.h>
#include <media/stagefright/foundation/hexdump.h>
#include <media/stagefright/DataSource.h>
#include <media/stagefright/MediaBuffer.h>
@@ -563,7 +564,12 @@
TRESPASS();
}
- if (srcOffset + mNALSizeLen + NALsize > srcSize) {
+ if (srcOffset + mNALSizeLen + NALsize <= srcOffset + mNALSizeLen) {
+ frame->release();
+ frame = NULL;
+
+ return ERROR_MALFORMED;
+ } else if (srcOffset + mNALSizeLen + NALsize > srcSize) {
break;
}
