shithub: openh264

Download patch

ref: d58148cb665905a29e579fe1ce980c4d9265d098
parent: 9a81539c5b758d745ddb9960ef714013cfaeb56f
author: wayne <huili2@cisco.com>
date: Wed Dec 26 10:10:53 EST 2018

fix low memory crash issue

--- a/test/BaseDecoderTest.h
+++ b/test/BaseDecoderTest.h
@@ -30,7 +30,7 @@
   BaseDecoderTest();
   int32_t SetUp();
   void TearDown();
-  void DecodeFile (const char* fileName, Callback* cbk);
+  bool DecodeFile (const char* fileName, Callback* cbk);
 
   bool Open (const char* fileName);
   bool DecodeNextFrame (Callback* cbk);
@@ -42,7 +42,7 @@
 
   std::ifstream file_;
   BufferedData buf_;
-  enum {
+  enum {\
     OpenFile,
     Decoding,
     EndOfStream,
--- a/test/api/BaseDecoderTest.cpp
+++ b/test/api/BaseDecoderTest.cpp
@@ -5,7 +5,7 @@
 #include "utils/BufferedData.h"
 #include "BaseDecoderTest.h"
 
-static void ReadFrame (std::ifstream* file, BufferedData* buf) {
+static bool ReadFrame (std::ifstream* file, BufferedData* buf) {
   // start code of a frame is {0, 0, 0, 1}
   int zeroCount = 0;
   char b;
@@ -14,10 +14,11 @@
   for (;;) {
     file->read (&b, 1);
     if (file->gcount() != 1) { // end of file
-      return;
+      return true;
     }
     if (!buf->PushBack (b)) {
-      FAIL() << "unable to allocate memory";
+      std::cout << "unable to allocate memory" << std::endl;
+      return false;
     }
 
     if (buf->Length() <= 4) {
@@ -29,10 +30,12 @@
     } else {
       if (b == 1) {
         if (file->seekg (-4, file->cur).good()) {
-          buf->SetLength (buf->Length() - 4);
-          return;
+          if (-1 == buf->SetLength(buf->Length() - 4))
+            return false;
+          return true;
         } else {
-          FAIL() << "unable to seek file";
+          std::cout << "unable to seek file" << std::endl;
+          return false;
         }
       } else if (b == 0) {
         zeroCount = 3;
@@ -145,15 +148,17 @@
     cbk->onDecodeFrame (frame);
   }
 }
-void BaseDecoderTest::DecodeFile (const char* fileName, Callback* cbk) {
+bool BaseDecoderTest::DecodeFile (const char* fileName, Callback* cbk) {
   std::ifstream file (fileName, std::ios::in | std::ios::binary);
-  ASSERT_TRUE (file.is_open());
+  if (!file.is_open())
+    return false;
 
   BufferedData buf;
   while (true) {
-    ReadFrame (&file, &buf);
+    if (false == ReadFrame(&file, &buf))
+      return false;
     if (::testing::Test::HasFatalFailure()) {
-      return;
+      return false;
     }
     if (buf.Length() == 0) {
       break;
@@ -160,7 +165,7 @@
     }
     DecodeFrame (buf.data(), buf.Length(), cbk);
     if (::testing::Test::HasFatalFailure()) {
-      return;
+      return false;
     }
   }
 
@@ -175,6 +180,7 @@
   for (int32_t i = 0; i < num_of_frames_in_buffer; ++i) {
     FlushFrame (cbk);
   }
+  return true;
 }
 
 bool BaseDecoderTest::Open (const char* fileName) {
@@ -191,7 +197,8 @@
 bool BaseDecoderTest::DecodeNextFrame (Callback* cbk) {
   switch (decodeStatus_) {
   case Decoding:
-    ReadFrame (&file_, &buf_);
+    if (false == ReadFrame(&file_, &buf_))
+      return false;
     if (::testing::Test::HasFatalFailure()) {
       return false;
     }
--- a/test/api/BaseEncoderTest.cpp
+++ b/test/api/BaseEncoderTest.cpp
@@ -96,7 +96,7 @@
 
   BufferedData buf;
   buf.SetLength (frameSize);
-  ASSERT_TRUE (buf.Length() == (size_t)frameSize);
+  ASSERT_TRUE (buf.Length() == (size_t)frameSize); //include memory fail (-1) case
 
   SFrameBSInfo info;
   memset (&info, 0, sizeof (SFrameBSInfo));
--- a/test/api/DataGenerator.cpp
+++ b/test/api/DataGenerator.cpp
@@ -17,7 +17,7 @@
   const int32_t kiFrameSize = SRC_FRAME_WIDTH * SRC_FRAME_HEIGHT;
   BufferedData sBuf;
   sBuf.SetLength (kiFrameSize);
-  if (sBuf.Length() != (size_t)kiFrameSize) {
+  if (sBuf.Length() != (size_t)kiFrameSize) { //include memory fail (-1) case
     return false;
   }
 
--- a/test/api/decoder_test.cpp
+++ b/test/api/decoder_test.cpp
@@ -76,9 +76,9 @@
   FileParam p = GetParam();
 #if defined(ANDROID_NDK)
   std::string filename = std::string ("/sdcard/") + p.fileName;
-  DecodeFile (filename.c_str(), this);
+  ASSERT_TRUE ( DecodeFile (filename.c_str(), this));
 #else
-  DecodeFile (p.fileName, this);
+  ASSERT_TRUE (DecodeFile(p.fileName, this));
 #endif
 
   unsigned char digest[SHA_DIGEST_LENGTH];
--- a/test/api/encode_decode_api_test.cpp
+++ b/test/api/encode_decode_api_test.cpp
@@ -34,7 +34,8 @@
   //for encoder
   //I420: 1(Y) + 1/4(U) + 1/4(V)
   int frameSize = EncDecFileParam.width * EncDecFileParam.height * 3 / 2;
-  buf_.SetLength (frameSize);
+  if (-1 == buf_.SetLength(frameSize))
+    return false;
   if (buf_.Length() != (size_t)frameSize) {
     printf ("buf_.Length() failed! frameSize = %d\n", frameSize);
     return false;
--- a/test/api/encode_options_test.cpp
+++ b/test/api/encode_options_test.cpp
@@ -16,7 +16,7 @@
   int frameSize = iWidth * iHeight * 3 / 2;
 
   buf_.SetLength (frameSize);
-  if (buf_.Length() != (size_t)frameSize) {
+  if (buf_.Length() != (size_t)frameSize) { //include memory fail (-1) case
     printf ("buf_.Length() failed! frameSize = %d\n", frameSize);
     return false;
   }
--- a/test/encoder/EncUT_InterfaceTest.cpp
+++ b/test/encoder/EncUT_InterfaceTest.cpp
@@ -68,7 +68,7 @@
 
   BufferedData buf;
   buf.SetLength (frameSize);
-  ASSERT_TRUE (buf.Length() == (size_t)frameSize);
+  ASSERT_TRUE (buf.Length() == (size_t)frameSize); //include memory fail (-1) case
 
   SFrameBSInfo info;
   memset (&info, 0, sizeof (SFrameBSInfo));
--- a/test/utils/BufferedData.h
+++ b/test/utils/BufferedData.h
@@ -36,7 +36,8 @@
     len = std::min (length_, len);
     memcpy (ptr, data_, len);
     memmove (data_, data_ + len, length_ - len);
-    SetLength (length_ - len);
+    if (-1 == SetLength (length_ - len))
+      return -1;
     return len;
   }
 
@@ -44,12 +45,17 @@
     length_ = 0;
   }
 
-  void SetLength (size_t newLen) {
+  int SetLength (size_t newLen) {
     if (EnsureCapacity (newLen)) {
       length_ = newLen;
     } else {
-      FAIL () << "unable to alloc memory in SetLength()";
+      Clear();
+      //FAIL () << "unable to alloc memory in SetLength()";
+      std::cout << "unable to alloc memory in SetLength()" << std::endl;
+      return -1;
     }
+
+    return 0;
   }
 
   size_t Length() const {