ref: 45c78f438e803f454cb30786e8c35a3c0969b3bd
dir: /module/gmp-openh264.cpp/
/*!
* \copy
* Copyright (c) 2009-2014, Cisco Systems
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*
*************************************************************************************
*/
#include <stdint.h>
#include <time.h>
#include <cstdio>
#include <cstring>
#include <iostream>
#include <memory>
#include <assert.h>
#include <limits.h>
#include "gmp-platform.h"
#include "gmp-video-host.h"
#include "gmp-video-encode.h"
#include "gmp-video-decode.h"
#include "gmp-video-frame-i420.h"
#include "gmp-video-frame-encoded.h"
#include "codec_def.h"
#include "codec_app_def.h"
#include "codec_api.h"
#include "task_utils.h"
#if defined(_MSC_VER)
#define PUBLIC_FUNC __declspec(dllexport)
#else
#define PUBLIC_FUNC
#endif
// This is for supporting older versions which do not have support for nullptr.
#if defined(__clang__)
# ifndef __has_extension
# define __has_extension __has_feature
# endif
# if __has_extension(cxx_nullptr)
# define GMP_HAVE_NULLPTR
# endif
#elif defined(__GNUC__)
# if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
# if (__GNU_C__ >=4)
# if (__GNU_C_MINOR__ >= 6)
# define GMP_HAVE_NULLPTR
# endif
# endif
# endif
#elif defined(_MSC_VER)
# define GMP_HAVE_NULLPTR
#endif
#if !defined (GMP_HAVE_NULLPTR)
# define nullptr __null
#endif
static int g_log_level = 0;
#define GMPLOG(l, x) do { \
if (l <= g_log_level) { \
const char *log_string = "unknown"; \
if ((l >= 0) && (l <= 3)) { \
log_string = kLogStrings[l]; \
} \
std::cerr << log_string << ": " << x << std::endl; \
} \
} while(0)
#define GL_CRIT 0
#define GL_ERROR 1
#define GL_INFO 2
#define GL_DEBUG 3
const char *kLogStrings[] = {
"Critical",
"Error",
"Info",
"Debug"
};
static GMPPlatformAPI* g_platform_api = nullptr;
class OpenH264VideoEncoder;
template <typename T> class SelfDestruct {
public:
SelfDestruct(T* t) : t_(t) {}
~SelfDestruct() {
if (t_) {
t_->Destroy();
}
}
T* forget() {
T* t = t_;
t_ = nullptr;
return t;
}
private:
T* t_;
};
class FrameStats {
public:
FrameStats(const char *type) :
frames_in_(0),
frames_out_(0),
start_time_(time(0)),
last_time_(start_time_),
type_(type) {}
void FrameIn() {
++frames_in_;
time_t now = time(0);
if (now == last_time_) {
return;
}
if (!(frames_in_ % 10)) {
GMPLOG(GL_INFO, type_ << ": " << now << " Frame count "
<< frames_in_
<< "(" << (frames_in_ / (now - start_time_)) << "/"
<< (30 / (now - last_time_)) << ")"
<< " -- " << frames_out_);
last_time_ = now;
}
}
void FrameOut() {
++frames_out_;
}
private:
uint64_t frames_in_;
uint64_t frames_out_;
time_t start_time_;
time_t last_time_;
const std::string type_;
};
class OpenH264VideoEncoder : public GMPVideoEncoder
{
public:
OpenH264VideoEncoder(GMPVideoHost *hostAPI) :
host_(hostAPI),
worker_thread_(nullptr),
encoder_(nullptr),
max_payload_size_(0),
callback_(nullptr),
stats_("Encoder") {}
virtual ~OpenH264VideoEncoder() {
worker_thread_->Join();
}
virtual GMPVideoErr InitEncode(const GMPVideoCodec& codecSettings,
GMPEncoderCallback* callback,
int32_t numberOfCores,
uint32_t maxPayloadSize) {
GMPErr err = g_platform_api->createthread(&worker_thread_);
if (err != GMPNoErr) {
GMPLOG(GL_ERROR, "Couldn't create new thread");
return GMPVideoGenericErr;
}
int rv = WelsCreateSVCEncoder(&encoder_);
if (rv) {
return GMPVideoGenericErr;
}
SEncParamBase param;
memset(¶m, 0, sizeof(param));
GMPLOG(GL_INFO, "Initializing encoder at "
<< codecSettings.mWidth
<< "x"
<< codecSettings.mHeight
<< "@"
<< static_cast<int>(codecSettings.mMaxFramerate)
<< "max payload size="
<< maxPayloadSize);
// Translate parameters.
param.iUsageType = CAMERA_VIDEO_REAL_TIME;
param.iPicWidth = codecSettings.mWidth;
param.iPicHeight = codecSettings.mHeight;
param.iTargetBitrate = codecSettings.mStartBitrate * 1000;
GMPLOG(GL_INFO, "Initializing Bit Rate at: Start: "
<< codecSettings.mStartBitrate
<< "; Min: "
<< codecSettings.mMinBitrate
<< "; Max: "
<< codecSettings.mMaxBitrate);
param.iRCMode = RC_BITRATE_MODE;
// TODO(ekr@rtfm.com). Scary conversion from unsigned char to float below.
param.fMaxFrameRate = static_cast<float>(codecSettings.mMaxFramerate);
param.iInputCsp = videoFormatI420;
rv = encoder_->Initialize(¶m);
if (rv) {
GMPLOG(GL_ERROR, "Couldn't initialize encoder");
return GMPVideoGenericErr;
}
max_payload_size_ = maxPayloadSize;
callback_ = callback;
GMPLOG(GL_INFO, "Initialized encoder");
return GMPVideoNoErr;
}
virtual GMPVideoErr Encode(GMPVideoi420Frame* inputImage,
const GMPCodecSpecificInfo& codecSpecificInfo,
const std::vector<GMPVideoFrameType>& frameTypes)
{
GMPLOG(GL_DEBUG,
__FUNCTION__
<< " size="
<< inputImage->Width() << "x" << inputImage->Height());
stats_.FrameIn();
assert(!frameTypes.empty());
if (frameTypes.empty()) {
GMPLOG(GL_ERROR, "No frame types provided");
return GMPVideoGenericErr;
}
worker_thread_->Post(WrapTask(
this, &OpenH264VideoEncoder::Encode_w,
inputImage,
(frameTypes)[0]));
return GMPVideoGenericErr;
}
void Encode_w(GMPVideoi420Frame* inputImage,
GMPVideoFrameType frame_type) {
SFrameBSInfo encoded;
SSourcePicture src;
src.iColorFormat = videoFormatI420;
src.iStride[0] = inputImage->Stride(kGMPYPlane);
src.pData[0] = reinterpret_cast<unsigned char*>(
const_cast<uint8_t *>(inputImage->Buffer(kGMPYPlane)));
src.iStride[1] = inputImage->Stride(kGMPUPlane);
src.pData[1] = reinterpret_cast<unsigned char*>(
const_cast<uint8_t *>(inputImage->Buffer(kGMPUPlane)));
src.iStride[2] = inputImage->Stride(kGMPVPlane);
src.pData[2] = reinterpret_cast<unsigned char*>(
const_cast<uint8_t *>(inputImage->Buffer(kGMPVPlane)));
src.iStride[3] = 0;
src.pData[3] = nullptr;
src.iPicWidth = inputImage->Width();
src.iPicHeight = inputImage->Height();
const SSourcePicture* pics = &src;
int result = encoder_->EncodeFrame(pics, &encoded);
if (result != cmResultSuccess) {
GMPLOG(GL_ERROR, "Couldn't encode frame. Error = " << result);
}
// Translate int to enum
GMPVideoFrameType encoded_type;
bool has_frame = false;
switch (encoded.eOutputFrameType) {
case videoFrameTypeIDR:
encoded_type = kGMPKeyFrame;
has_frame = true;
break;
case videoFrameTypeI:
encoded_type = kGMPKeyFrame;
has_frame = true;
break;
case videoFrameTypeP:
encoded_type = kGMPDeltaFrame;
has_frame = true;
break;
case videoFrameTypeSkip:
// Can skip the call back since no actual bitstream will be generated
break;
case videoFrameTypeIPMixed://this type is currently not suppported
case videoFrameTypeInvalid:
GMPLOG(GL_ERROR, "Couldn't encode frame. Type = "
<< encoded.eOutputFrameType);
break;
default:
// The API is defined as returning a type.
assert(false);
break;
}
if (!has_frame) {
return;
}
// Synchronously send this back to the main thread for delivery.
g_platform_api->syncrunonmainthread(WrapTask(
this,
&OpenH264VideoEncoder::Encode_m,
inputImage,
&encoded,
encoded_type));
}
void Encode_m(GMPVideoi420Frame* frame, SFrameBSInfo* encoded,
GMPVideoFrameType frame_type) {
// Now return the encoded data back to the parent.
GMPVideoFrame* ftmp;
GMPVideoErr err = host_->CreateFrame(kGMPEncodedVideoFrame, &ftmp);
if (err != GMPVideoNoErr) {
GMPLOG(GL_ERROR, "Error creating encoded frame");
return;
}
GMPVideoEncodedFrame* f = static_cast<GMPVideoEncodedFrame*>(ftmp);
// Buffer up the data.
uint32_t length = 0;
std::vector<uint32_t> lengths;
for (int i=0; i<encoded->iLayerNum; ++i) {
lengths.push_back(0);
for (int j=0; j<encoded->sLayerInfo[i].iNalCount; ++j) {
lengths[i] += encoded->sLayerInfo[i].pNalLengthInByte[j];
length += encoded->sLayerInfo[i].pNalLengthInByte[j];
}
}
// TODO start-code to length conversion here when gmp
// stops doing it for us before this call.
err = f->CreateEmptyFrame(length);
if (err != GMPVideoNoErr) {
GMPLOG(GL_ERROR, "Error allocating frame data");
f->Destroy();
return;
}
// Copy the data.
// Here we concatenate into one big buffer
uint8_t* tmp = f->Buffer();
for (int i=0; i<encoded->iLayerNum; ++i) {
memcpy(tmp, encoded->sLayerInfo[i].pBsBuf, lengths[i]);
tmp += lengths[i];
}
f->SetEncodedWidth(frame->Width());
f->SetEncodedHeight(frame->Height());
f->SetTimeStamp(frame->Timestamp());
f->SetFrameType(frame_type);
f->SetCompleteFrame(true);
GMPLOG(GL_DEBUG, "Encoding complete. type= "
<< f->FrameType()
<< " length="
<< f->Size()
<< " timestamp="
<< f->TimeStamp());
// Destroy the frame.
frame->Destroy();
// Return the encoded frame.
GMPCodecSpecificInfo info;
memset(&info, 0, sizeof(info));
// TODO need to set what goes in this info structure.
callback_->Encoded(f, info);
stats_.FrameOut();
}
virtual GMPVideoErr SetChannelParameters(uint32_t aPacketLoss, uint32_t aRTT) {
return GMPVideoNoErr;
}
virtual GMPVideoErr SetRates(uint32_t aNewBitRate, uint32_t aFrameRate) {
GMPLOG(GL_INFO, "[SetRates] Begin with: "
<< aNewBitRate << " , "<< aFrameRate);
//update bitrate if needed
const int32_t newBitRate = aNewBitRate*1000; //kbps->bps
SBitrateInfo existEncoderBitRate;
existEncoderBitRate.iLayer = SPATIAL_LAYER_ALL;
int rv = encoder_->GetOption(ENCODER_OPTION_BITRATE, &existEncoderBitRate);
if (rv!=cmResultSuccess) {
GMPLOG(GL_ERROR, "[SetRates] Error in Getting Bit Rate at Layer:"
<< rv
<< " ; Layer = "
<< existEncoderBitRate.iLayer
<< " ; BR = "
<< existEncoderBitRate.iBitrate);
return GMPVideoGenericErr;
}
if ( rv==cmResultSuccess && existEncoderBitRate.iBitrate!=newBitRate ) {
SBitrateInfo newEncoderBitRate;
newEncoderBitRate.iLayer = SPATIAL_LAYER_ALL;
newEncoderBitRate.iBitrate = newBitRate;
rv = encoder_->SetOption(ENCODER_OPTION_BITRATE, &newEncoderBitRate);
if (rv==cmResultSuccess) {
GMPLOG(GL_INFO, "[SetRates] Update Encoder Bandwidth (AllLayers): ReturnValue: "
<< rv
<< "BitRate(kbps): "
<< aNewBitRate);
} else {
GMPLOG(GL_ERROR, "[SetRates] Error in Setting Bit Rate at Layer:"
<< rv
<< " ; Layer = "
<< newEncoderBitRate.iLayer
<< " ; BR = "
<< newEncoderBitRate.iBitrate);
return GMPVideoGenericErr;
}
}
//update framerate if needed
float existFrameRate = 0;
rv = encoder_->GetOption(ENCODER_OPTION_FRAME_RATE, &existFrameRate);
if (rv!=cmResultSuccess) {
GMPLOG(GL_ERROR, "[SetRates] Error in Getting Frame Rate:"
<< rv << " FrameRate: " << existFrameRate);
return GMPVideoGenericErr;
}
if ( rv==cmResultSuccess &&
( aFrameRate-existFrameRate > 0.001f ||
existFrameRate-aFrameRate > 0.001f ) ) {
float newFrameRate = static_cast<float>(aFrameRate);
rv = encoder_->SetOption(ENCODER_OPTION_FRAME_RATE, &newFrameRate);
if (rv==cmResultSuccess) {
GMPLOG(GL_INFO, "[SetRates] Update Encoder Frame Rate: ReturnValue: "
<< rv << " FrameRate: " << aFrameRate);
} else {
GMPLOG(GL_ERROR, "[SetRates] Error in Setting Frame Rate: ReturnValue: "
<< rv << " FrameRate: " << aFrameRate);
return GMPVideoGenericErr;
}
}
return GMPVideoNoErr;
}
virtual GMPVideoErr SetPeriodicKeyFrames(bool aEnable) {
return GMPVideoNoErr;
}
virtual void EncodingComplete() {
delete this;
}
private:
GMPVideoHost* host_;
GMPThread* worker_thread_;
ISVCEncoder* encoder_;
uint32_t max_payload_size_;
GMPEncoderCallback* callback_;
FrameStats stats_;
};
class OpenH264VideoDecoder : public GMPVideoDecoder {
public:
OpenH264VideoDecoder(GMPVideoHost *hostAPI) :
host_(hostAPI),
worker_thread_(nullptr),
callback_(nullptr),
decoder_(nullptr),
stats_("Decoder") {}
virtual ~OpenH264VideoDecoder() {
}
virtual GMPVideoErr InitDecode(const GMPVideoCodec& codecSettings,
GMPDecoderCallback* callback,
int32_t coreCount) {
GMPLOG(GL_INFO, "InitDecode");
GMPErr err = g_platform_api->createthread(&worker_thread_);
if (err != GMPNoErr) {
GMPLOG(GL_ERROR, "Couldn't create new thread");
return GMPVideoGenericErr;
}
if (WelsCreateDecoder(&decoder_)) {
GMPLOG(GL_ERROR, "Couldn't create decoder");
return GMPVideoGenericErr;
}
if (!decoder_) {
GMPLOG(GL_ERROR, "Couldn't create decoder");
return GMPVideoGenericErr;
}
SDecodingParam param;
memset(¶m, 0, sizeof(param));
param.iOutputColorFormat = videoFormatI420;
param.uiTargetDqLayer = UCHAR_MAX; // Default value
param.uiEcActiveFlag = 1; // Error concealment on.
param.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_DEFAULT;
if (decoder_->Initialize(¶m)) {
GMPLOG(GL_ERROR, "Couldn't initialize decoder");
return GMPVideoGenericErr;
}
callback_ = callback;
return GMPVideoNoErr;
}
virtual GMPVideoErr Decode(GMPVideoEncodedFrame* inputFrame,
bool missingFrames,
const GMPCodecSpecificInfo& codecSpecificInfo,
int64_t renderTimeMs = -1) {
GMPLOG(GL_DEBUG, __FUNCTION__
<< "Decoding frame size=" << inputFrame->Size()
<< " timestamp=" << inputFrame->TimeStamp());
stats_.FrameIn();
worker_thread_->Post(WrapTask(
this, &OpenH264VideoDecoder::Decode_w,
inputFrame,
missingFrames,
renderTimeMs));
return GMPVideoNoErr;
}
virtual GMPVideoErr Reset() {
return GMPVideoNoErr;
}
virtual GMPVideoErr Drain() {
return GMPVideoNoErr;
}
virtual void DecodingComplete() {
delete this;
}
private:
void Decode_w(GMPVideoEncodedFrame* inputFrame,
bool missingFrames,
int64_t renderTimeMs = -1) {
GMPLOG(GL_DEBUG, "Frame decode on worker thread length = "
<< inputFrame->Size());
SBufferInfo decoded;
bool valid = false;
memset(&decoded, 0, sizeof(decoded));
unsigned char *data[3] = {nullptr, nullptr, nullptr};
int rv = decoder_->DecodeFrame2(inputFrame->Buffer(),
inputFrame->Size(),
data,
&decoded);
if (rv) {
GMPLOG(GL_ERROR, "Decoding error rv=" << rv);
} else {
valid = true;
}
g_platform_api->syncrunonmainthread(WrapTask(
this,
&OpenH264VideoDecoder::Decode_m,
inputFrame,
&decoded,
data,
renderTimeMs,
valid));
}
// Return the decoded data back to the parent.
void Decode_m(GMPVideoEncodedFrame* inputFrame,
SBufferInfo* decoded,
unsigned char* data[3],
int64_t renderTimeMs,
bool valid) {
// Attach a self-destructor so that this dies on return.
SelfDestruct<GMPVideoEncodedFrame> ifd(inputFrame);
// If we don't actually have data, just abort.
if (!valid) {
return;
}
if (decoded->iBufferStatus != 1) {
return;
}
int width = decoded->UsrData.sSystemBuffer.iWidth;
int height = decoded->UsrData.sSystemBuffer.iHeight;
int ystride = decoded->UsrData.sSystemBuffer.iStride[0];
int uvstride = decoded->UsrData.sSystemBuffer.iStride[1];
GMPLOG(GL_DEBUG, "Video frame ready for display "
<< width
<< "x"
<< height
<< " timestamp="
<< inputFrame->TimeStamp());
GMPVideoFrame* ftmp = nullptr;
// Translate the image.
GMPVideoErr err = host_->CreateFrame(kGMPI420VideoFrame, &ftmp);
if (err != GMPVideoNoErr) {
GMPLOG(GL_ERROR, "Couldn't allocate empty I420 frame");
return;
}
GMPVideoi420Frame* frame = static_cast<GMPVideoi420Frame*>(ftmp);
err = frame->CreateFrame(
ystride * height, static_cast<uint8_t *>(data[0]),
uvstride * height/2, static_cast<uint8_t *>(data[1]),
uvstride * height/2, static_cast<uint8_t *>(data[2]),
width, height,
ystride, uvstride, uvstride);
if (err != GMPVideoNoErr) {
GMPLOG(GL_ERROR, "Couldn't make decoded frame");
return;
}
GMPLOG(GL_DEBUG, "Allocated size = "
<< frame->AllocatedSize(kGMPYPlane));
frame->SetTimestamp(inputFrame->TimeStamp());
frame->SetRenderTime_ms(renderTimeMs);
callback_->Decoded(frame);
stats_.FrameOut();
}
GMPVideoHost* host_;
GMPThread* worker_thread_;
GMPDecoderCallback* callback_;
ISVCDecoder* decoder_;
FrameStats stats_;
};
extern "C" {
PUBLIC_FUNC GMPErr
GMPInit(GMPPlatformAPI* aPlatformAPI) {
g_platform_api = aPlatformAPI;
return GMPNoErr;
}
PUBLIC_FUNC GMPErr
GMPGetAPI(const char* aApiName, void* aHostAPI, void** aPluginApi) {
if (!strcmp(aApiName, "decode-video")) {
*aPluginApi = new OpenH264VideoDecoder(static_cast<GMPVideoHost*>(aHostAPI));
return GMPNoErr;
} else if (!strcmp(aApiName, "encode-video")) {
*aPluginApi = new OpenH264VideoEncoder(static_cast<GMPVideoHost*>(aHostAPI));
return GMPNoErr;
}
return GMPGenericErr;
}
PUBLIC_FUNC void
GMPShutdown(void) {
g_platform_api = nullptr;
}
} // extern "C"