shithub: openh264

--- a/codec/decoder/core/inc/fmo.h

+++ b/codec/decoder/core/inc/fmo.h

@@ -97,7 +97,7 @@

  * \return  true - update/insert successfully; false - failed;

*/

-bool FmoParamUpdate (PFmo pFmo, PSps pSps, PPps pPps, int32_t* pActiveFmoNum, CMemoryAlign* pMa);

+int32_t FmoParamUpdate (PFmo pFmo, PSps pSps, PPps pPps, int32_t* pActiveFmoNum, CMemoryAlign* pMa);

/*!

  * \brief   Get successive mb to be processed with given current mb_xy

--- a/codec/decoder/core/src/au_parser.cpp

+++ b/codec/decoder/core/src/au_parser.cpp

@@ -1148,7 +1148,9 @@

   if (pCtx->pParam->bParseOnly) {

     if (kSrcNalLen >= SPS_PPS_BS_SIZE - 4) { //sps bs exceeds!

-      pCtx->iErrorCode |= dsOutOfMemory;

+      WelsLog (& (pCtx->sLogCtx), WELS_LOG_WARNING, "sps payload size (%d) too large for parse only (%d), not supported!",

+               kSrcNalLen, SPS_PPS_BS_SIZE - 4);

+      pCtx->iErrorCode |= dsBitstreamError;

       return GENERATE_ERROR_NO (ERR_LEVEL_PARAM_SETS, ERR_INFO_OUT_OF_MEMORY);

     if (!kbUseSubsetFlag) { //SPS

@@ -1181,6 +1183,7 @@

       uint8_t* pBsBuf = static_cast<uint8_t*> (pMa->WelsMallocz (SPS_PPS_BS_SIZE + 4,

                         "Temp buffer for parse only usage.")); //to reserve 4 bytes for UVLC writing buffer

       if (NULL == pBsBuf) {

+        WelsLog (& (pCtx->sLogCtx), WELS_LOG_ERROR, "sps buffer alloc failed for parse only!");

         pCtx->iErrorCode |= dsOutOfMemory;

         return pCtx->iErrorCode;

@@ -1447,7 +1450,9 @@

   if (pCtx->pParam->bParseOnly) {

     if (kSrcNalLen >= SPS_PPS_BS_SIZE - 4) { //pps bs exceeds

-      pCtx->iErrorCode |= dsOutOfMemory;

+      WelsLog (& (pCtx->sLogCtx), WELS_LOG_WARNING, "pps payload size (%d) too large for parse only (%d), not supported!",

+               kSrcNalLen, SPS_PPS_BS_SIZE - 4);

+      pCtx->iErrorCode |= dsBitstreamError;

       return GENERATE_ERROR_NO (ERR_LEVEL_PARAM_SETS, ERR_INFO_OUT_OF_MEMORY);

     SPpsBsInfo* pPpsBs = &pCtx->sPpsBsInfo [uiPpsId];

--- a/codec/decoder/core/src/decoder.cpp

+++ b/codec/decoder/core/src/decoder.cpp

@@ -475,7 +475,7 @@

/*!

  * \brief   Open decoder

*/

-int32_t WelsOpenDecoder (PWelsDecoderContext pCtx) {

+int32_t WelsOpenDecoder (PWelsDecoderContext pCtx, SLogContext* pLogCtx) {

   int iRet = ERR_NONE;

   // function pointers

   InitDecFuncs (pCtx, pCtx->uiCpuFlag);

@@ -485,8 +485,11 @@

   // static memory

   iRet = WelsInitStaticMemory (pCtx);

-  if (ERR_NONE != iRet)

+  if (ERR_NONE != iRet) {

+    pCtx->iErrorCode |= dsOutOfMemory;

+    WelsLog (pLogCtx, WELS_LOG_ERROR, "WelsInitStaticMemory() failed in WelsOpenDecoder().");

     return iRet;

+  }

 #ifdef LONG_TERM_REF

   pCtx->bParamSetsLostFlag = true;

@@ -568,7 +571,7 @@

   // open decoder

-  return WelsOpenDecoder (pCtx);

+  return WelsOpenDecoder (pCtx, pLogCtx);

/*!

@@ -817,17 +820,17 @@

   bool bReallocFlag = false;

   iErr = WelsRequestMem (pCtx, kiMbWidth, kiMbHeight, bReallocFlag); // common memory used

   if (ERR_NONE != iErr) {

-    WelsLog (& (pCtx->sLogCtx), WELS_LOG_WARNING,

+    WelsLog (& (pCtx->sLogCtx), WELS_LOG_ERROR,

              "SyncPictureResolutionExt()::WelsRequestMem--buffer allocated failure.");

-    pCtx->iErrorCode = dsOutOfMemory;

+    pCtx->iErrorCode |= dsOutOfMemory;

     return iErr;

   iErr = InitialDqLayersContext (pCtx, kiPicWidth, kiPicHeight);

   if (ERR_NONE != iErr) {

-    WelsLog (& (pCtx->sLogCtx), WELS_LOG_WARNING,

+    WelsLog (& (pCtx->sLogCtx), WELS_LOG_ERROR,

              "SyncPictureResolutionExt()::InitialDqLayersContext--buffer allocated failure.");

-    pCtx->iErrorCode = dsOutOfMemory;

+    pCtx->iErrorCode |= dsOutOfMemory;

 #if defined(MEMORY_MONITOR)

   if (bReallocFlag) {

--- a/codec/decoder/core/src/decoder_core.cpp

+++ b/codec/decoder/core/src/decoder_core.cpp

@@ -106,7 +106,7 @@

           pSpsBs = bSubSps ? &pCtx->sSubsetSpsBsInfo [iSpsId] : &pCtx->sSpsBsInfo [iSpsId];

           pPpsBs = &pCtx->sPpsBsInfo [iPpsId];

           if (pDstBuf - pParser->pDstBuff + pSpsBs->uiSpsBsLen + pPpsBs->uiPpsBsLen >= MAX_ACCESS_UNIT_CAPACITY) {

-            WelsLog (& (pCtx->sLogCtx), WELS_LOG_WARNING,

+            WelsLog (& (pCtx->sLogCtx), WELS_LOG_ERROR,

                      "DecodeFrameConstruction(): sps pps size: (%d %d) too large. Failed to parse. \n", pSpsBs->uiSpsBsLen,

                      pPpsBs->uiPpsBsLen);

             pCtx->iErrorCode |= dsOutOfMemory;

@@ -130,7 +130,7 @@

         pNalBs = pCurNal->sNalData.sVclNal.pNalPos;

         pParser->iNalLenInByte [pParser->iNalNum ++] = iNalLen;

         if (pDstBuf - pParser->pDstBuff + iNalLen >= MAX_ACCESS_UNIT_CAPACITY) {

-          WelsLog (& (pCtx->sLogCtx), WELS_LOG_WARNING,

+          WelsLog (& (pCtx->sLogCtx), WELS_LOG_ERROR,

                    "DecodeFrameConstruction(): composed output size (%ld) exceeds (%d). Failed to parse. \n",

                    (long) (pDstBuf - pParser->pDstBuff + iNalLen), MAX_ACCESS_UNIT_CAPACITY);

           pCtx->iErrorCode |= dsOutOfMemory;

@@ -517,8 +517,11 @@

   //Realloc sRawData

   uint8_t* pNewBsBuff = static_cast<uint8_t*> (pMa->WelsMallocz (iNewBuffLen, "pCtx->sRawData.pHead"));

-  if (pNewBsBuff == NULL)

+  if (pNewBsBuff == NULL) {

+    WelsLog (& (pCtx->sLogCtx), WELS_LOG_ERROR, "ExpandBsBuffer() Failed for malloc pNewBsBuff (%d)", iNewBuffLen);

+    pCtx->iErrorCode |= dsOutOfMemory;

     return ERR_INFO_OUT_OF_MEMORY;

+  }

   //Calculate and set the bs start and end position

   for (uint32_t i = 0; i <= pCtx->pAccessUnitList->uiActualUnitsNum; i++) {

@@ -539,8 +542,11 @@

   if (pCtx->pParam->bParseOnly) {

     //Realloc sSavedData

     uint8_t* pNewSavedBsBuff = static_cast<uint8_t*> (pMa->WelsMallocz (iNewBuffLen, "pCtx->sSavedData.pHead"));

-    if (pNewSavedBsBuff == NULL)

+    if (pNewSavedBsBuff == NULL) {

+      WelsLog (& (pCtx->sLogCtx), WELS_LOG_ERROR, "ExpandBsBuffer() Failed for malloc pNewSavedBsBuff (%d)", iNewBuffLen);

+      pCtx->iErrorCode |= dsOutOfMemory;

       return ERR_INFO_OUT_OF_MEMORY;

+    }

     //Copy current buffer status to new buffer

     memcpy (pNewSavedBsBuff, pCtx->sSavedData.pHead, pCtx->iMaxBsBufferSizeInByte);

@@ -2250,10 +2256,16 @@

       pLayerInfo.pSubsetSps = pShExt->pSubsetSps;

       pCtx->pFmo = &pCtx->sFmoList[iPpsId];

-      if (!FmoParamUpdate (pCtx->pFmo, pLayerInfo.pSps, pLayerInfo.pPps, &pCtx->iActiveFmoNum, pCtx->pMemAlign)) {

-        pCtx->iErrorCode |= dsBitstreamError;

-        WelsLog (& (pCtx->sLogCtx), WELS_LOG_WARNING, "DecodeCurrentAccessUnit(), FmoParamUpdate failed, eSliceType: %d.",

-                 pSh->eSliceType);

+      iRet = FmoParamUpdate (pCtx->pFmo, pLayerInfo.pSps, pLayerInfo.pPps, &pCtx->iActiveFmoNum, pCtx->pMemAlign);

+      if (ERR_NONE != iRet) {

+        if (iRet == ERR_INFO_OUT_OF_MEMORY) {

+          pCtx->iErrorCode |= dsOutOfMemory;

+          WelsLog (&(pCtx->sLogCtx), WELS_LOG_ERROR, "DecodeCurrentAccessUnit(), Fmo param alloc failed");

+        } else {

+          pCtx->iErrorCode |= dsBitstreamError;

+          WelsLog(&(pCtx->sLogCtx), WELS_LOG_WARNING, "DecodeCurrentAccessUnit(), FmoParamUpdate failed, eSliceType: %d.",

+            pSh->eSliceType);

+        }

         return GENERATE_ERROR_NO (ERR_LEVEL_SLICE_HEADER, ERR_INFO_FMO_INIT_FAIL);

--- a/codec/decoder/core/src/fmo.cpp

+++ b/codec/decoder/core/src/fmo.cpp

@@ -259,26 +259,20 @@

  * \return  true - update/insert successfully; false - failed;

*/

-bool FmoParamUpdate (PFmo pFmo, PSps pSps, PPps pPps, int32_t* pActiveFmoNum, CMemoryAlign* pMa) {

+int32_t FmoParamUpdate (PFmo pFmo, PSps pSps, PPps pPps, int32_t* pActiveFmoNum, CMemoryAlign* pMa) {

   const uint32_t kuiMbWidth = pSps->iMbWidth;

   const uint32_t kuiMbHeight = pSps->iMbHeight;

+  int32_t iRet = ERR_NONE;

+  if (FmoParamSetsChanged (pFmo, kuiMbWidth * kuiMbHeight, pPps->uiSliceGroupMapType, pPps->uiNumSliceGroups)) {

+    iRet = InitFmo (pFmo, pPps, kuiMbWidth, kuiMbHeight, pMa);

+    WELS_VERIFY_RETURN_IF (iRet, iRet);

-  if (FmoParamSetsChanged (pFmo,

-                           kuiMbWidth * kuiMbHeight,

-                           pPps->uiSliceGroupMapType,

-                           pPps->uiNumSliceGroups)) {

-    if (InitFmo (pFmo, pPps, kuiMbWidth, kuiMbHeight, pMa)) {

-      return false;

-    } else {

-      if (!pFmo->bActiveFlag && *pActiveFmoNum < MAX_PPS_COUNT) {

-        ++ (*pActiveFmoNum);

-        pFmo->bActiveFlag = true;

-      }

+    if (!pFmo->bActiveFlag && *pActiveFmoNum < MAX_PPS_COUNT) {

+      ++ (*pActiveFmoNum);

+      pFmo->bActiveFlag = true;

-  return true;

+  return iRet;

/*!

--- a/codec/decoder/plus/src/welsDecoderExt.cpp

+++ b/codec/decoder/plus/src/welsDecoderExt.cpp

@@ -264,7 +264,8 @@

   WELS_VERIFY_RETURN_IFNEQ (iRet, cmResultSuccess);

   //init decoder

-  WELS_VERIFY_RETURN_PROC_IF (cmMallocMemeError, WelsInitDecoder (m_pDecContext, &m_pWelsTrace->m_sLogCtx), UninitDecoder())

+  WELS_VERIFY_RETURN_PROC_IF (cmMallocMemeError, WelsInitDecoder (m_pDecContext, &m_pWelsTrace->m_sLogCtx),

+                              UninitDecoder())

   return cmResultSuccess;

@@ -678,6 +679,12 @@

     m_pDecContext->uiTimeStamp = 0;

   WelsDecodeBs (m_pDecContext, kpSrc, kiSrcLen, NULL, NULL, pDstInfo);

+  if (m_pDecContext->iErrorCode & dsOutOfMemory) {

+    if (ResetDecoder())

+      return dsOutOfMemory;

+    return dsErrorFree;

+  }

   if (!m_pDecContext->bFramePending && m_pDecContext->pParserBsInfo->iNalNum) {

     memcpy (pDstInfo, m_pDecContext->pParserBsInfo, sizeof (SParserBsInfo));