codecs: check OMX buffer size before use in (h263|h264)dec

Bug: 27833616
Change-Id: I0fd599b3da431425d89236ffdd9df423c11947c0

media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp

@@ -221,6 +221,14 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) {
         int32_t bufferSize = inHeader->nFilledLen;
         int32_t tmp = bufferSize;
 
+        OMX_U32 frameSize = (mWidth * mHeight * 3) / 2;
+        if (outHeader->nAllocLen < frameSize) {
+            android_errorWriteLog(0x534e4554, "27833616");
+            ALOGE("Insufficient output buffer size");
+            notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+            mSignalledError = true;
+            return;
+        }
         // The PV decoder is lying to us, sometimes it'll claim to only have
         // consumed a subset of the buffer when it clearly consumed all of it.
         // ignore whatever it says...
@@ -264,7 +272,7 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) {
         ++mInputBufferCount;
 
         outHeader->nOffset = 0;
-        outHeader->nFilledLen = (mWidth * mHeight * 3) / 2;
+        outHeader->nFilledLen = frameSize;
 
         List<BufferInfo *>::iterator it = outQueue.begin();
         while ((*it)->mHeader != outHeader) {

media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp

@@ -200,7 +200,12 @@ void SoftAVC::onQueueFilled(OMX_U32 /* portIndex */) {
         }
 
         if (mFirstPicture && !outQueue.empty()) {
-            drainOneOutputBuffer(mFirstPictureId, mFirstPicture);
+            if (!drainOneOutputBuffer(mFirstPictureId, mFirstPicture)) {
+                ALOGE("Drain failed");
+                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+                mSignalledError = true;
+                return;
+            }
             delete[] mFirstPicture;
             mFirstPicture = NULL;
             mFirstPictureId = -1;
@@ -240,15 +245,20 @@ void SoftAVC::saveFirstOutputBuffer(int32_t picId, uint8_t *data) {
     memcpy(mFirstPicture, data, pictureSize);
 }
 
-void SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) {
+bool SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) {
     List<BufferInfo *> &outQueue = getPortQueue(kOutputPortIndex);
     BufferInfo *outInfo = *outQueue.begin();
-    outQueue.erase(outQueue.begin());
     OMX_BUFFERHEADERTYPE *outHeader = outInfo->mHeader;
+    OMX_U32 frameSize = mWidth * mHeight * 3 / 2;
+    if (outHeader->nAllocLen - outHeader->nOffset < frameSize) {
+        android_errorWriteLog(0x534e4554, "27833616");
+        return false;
+    }
+    outQueue.erase(outQueue.begin());
     OMX_BUFFERHEADERTYPE *header = mPicToHeaderMap.valueFor(picId);
     outHeader->nTimeStamp = header->nTimeStamp;
     outHeader->nFlags = header->nFlags;
-    outHeader->nFilledLen = mWidth * mHeight * 3 / 2;
+    outHeader->nFilledLen = frameSize;
 
     uint8_t *dst = outHeader->pBuffer + outHeader->nOffset;
     const uint8_t *srcY = data;
@@ -263,6 +273,7 @@ void SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) {
     delete header;
     outInfo->mOwnedByUs = false;
     notifyFillBufferDone(outHeader);
+    return true;
 }
 
 void SoftAVC::drainAllOutputBuffers(bool eos) {
@@ -275,7 +286,12 @@ void SoftAVC::drainAllOutputBuffers(bool eos) {
                     mHandle, &decodedPicture, eos /* flush */)) {
             int32_t picId = decodedPicture.picId;
             uint8_t *data = (uint8_t *) decodedPicture.pOutputPicture;
-            drainOneOutputBuffer(picId, data);
+            if (!drainOneOutputBuffer(picId, data)) {
+                ALOGE("Drain failed");
+                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+                mSignalledError = true;
+                return;
+            }
         }
     }
 

media/libstagefright/codecs/on2/h264dec/SoftAVC.h

@@ -71,7 +71,7 @@ private:
 
     status_t initDecoder();
     void drainAllOutputBuffers(bool eos);
-    void drainOneOutputBuffer(int32_t picId, uint8_t *data);
+    bool drainOneOutputBuffer(int32_t picId, uint8_t *data);
     void saveFirstOutputBuffer(int32_t pidId, uint8_t *data);
     CropSettingsMode handleCropParams(const H264SwDecInfo& decInfo);