ref: 73e0ce9f876fc6ea042072addeb21cd9cbac443b
parent: 7d0cb59987f5c5c70cf55e35e65aa6b7a5bc460e
author: Sigrid Haflínudóttir <ftrvxmtrx@gmail.com>
date: Wed Sep 16 07:50:48 EDT 2020
report more errors; give up on non-baseline profile immediately
--- a/src/h264bsd_byte_stream.c
+++ b/src/h264bsd_byte_stream.c
@@ -117,6 +117,7 @@
{/* no start code prefix found -> error */
*readBytes = len;
+ werrstr("no start code prefix");return(HANTRO_NOK);
}
@@ -187,6 +188,7 @@
if (invalidStream)
{+ werrstr("invalid stream");return(HANTRO_NOK);
}
@@ -204,8 +206,10 @@
* following bytes: 0x00, 0x01, 0x02, 0x03. This implies that
* emulation prevention 0x03 byte shall not be the last byte
* of the stream. */
- if ( (i == 0) || (*(readPtr+1) > 0x03) )
+ if ( (i == 0) || (*(readPtr+1) > 0x03) ) {+ werrstr("emulation byte last in the stream");return(HANTRO_NOK);
+ }
/* do not write emulation prevention byte */
readPtr++;
@@ -215,8 +219,10 @@
{/* NAL unit shall not contain byte sequences 0x000000,
* 0x000001 or 0x000002 */
- if ( (zeroCount == 2) && (*readPtr <= 0x02) )
+ if ( (zeroCount == 2) && (*readPtr <= 0x02) ) {+ werrstr("start sequence in NAL unit");return(HANTRO_NOK);
+ }
if (*readPtr == 0)
zeroCount++;
--- a/src/h264bsd_cavlc.c
+++ b/src/h264bsd_cavlc.c
@@ -784,8 +784,10 @@
BUFFER_FLUSH(bufferValue, bufferBits, LENGTH_TC(tmp));
totalCoeff = TOTAL_COEFF(tmp);
- if (totalCoeff > maxNumCoeff)
+ if (totalCoeff > maxNumCoeff) {+ werrstr("invalid total coeff: %d", totalCoeff);return(HANTRO_NOK);
+ }
trailingOnes = TRAILING_ONES(tmp);
if (totalCoeff != 0)
@@ -813,8 +815,10 @@
{BUFFER_SHOW(bufferValue, bufferBits, bit, 16);
levelPrefix = DecodeLevelPrefix(bit);
- if (levelPrefix == VLC_NOT_FOUND)
+ if (levelPrefix == VLC_NOT_FOUND) {+ werrstr("level prefix unknown");return(HANTRO_NOK);
+ }
BUFFER_FLUSH(bufferValue, bufferBits, levelPrefix+1);
if (levelPrefix < 14)
@@ -865,8 +869,10 @@
BUFFER_SHOW(bufferValue, bufferBits, bit,9);
zerosLeft = DecodeTotalZeros(bit, totalCoeff,
(u32)(maxNumCoeff == 4));
- if (!zerosLeft)
+ if (!zerosLeft) {+ werrstr("no zeros left");return(HANTRO_NOK);
+ }
BUFFER_FLUSH(bufferValue, bufferBits, LENGTH(zerosLeft));
zerosLeft = INFO(zerosLeft);
}
--- a/src/h264bsd_decoder.c
+++ b/src/h264bsd_decoder.c
@@ -103,8 +103,10 @@
size = (sizeof(macroblockLayer_t) + 63) & ~0x3F;
pStorage->mbLayer = (macroblockLayer_t*)malloc(size);
- if (!pStorage->mbLayer)
+ if (!pStorage->mbLayer) {+ werrstr("no memory");return HANTRO_NOK;
+ }
if (noOutputReordering)
pStorage->noReordering = HANTRO_TRUE;
--- a/src/h264bsd_dpb.c
+++ b/src/h264bsd_dpb.c
@@ -285,8 +285,10 @@
}
/* find corresponding picture from dpb */
index = FindDpbPic(dpb, picNum, isShortTerm);
- if (index < 0 || !IS_EXISTING(dpb->buffer[index]))
+ if (index < 0 || !IS_EXISTING(dpb->buffer[index])) {+ werrstr("dpb: picture not found");return(HANTRO_NOK);
+ }
/* shift pictures */
for (j = numRefIdxActive; j > refIdx; j--)
@@ -652,7 +654,7 @@
if (image->data != dpb->currentOut->data)
{- EPRINT("TRYING TO MARK NON-ALLOCATED IMAGE");+ werrstr("trying to mark non-allocated image");return(HANTRO_NOK);
}
@@ -760,6 +762,7 @@
break;
default: /* invalid memory management control operation */
+ werrstr("invalid mem management control op %d", mark->operation[i].memoryManagementControlOperation);status = HANTRO_NOK;
break;
}
@@ -792,6 +795,7 @@
/* no room */
else
{+ werrstr("no room for pic in dpb");status = HANTRO_NOK;
}
}
--- a/src/h264bsd_inter_prediction.c
+++ b/src/h264bsd_inter_prediction.c
@@ -535,13 +535,17 @@
mv.ver += mvPred.ver;
/* horizontal motion vector range [-2048, 2047.75] */
- if ((u32)(i32)(mv.hor+8192) >= (16384))
+ if ((u32)(i32)(mv.hor+8192) >= (16384)) {+ werrstr("horizontal motion vector out of range");return(HANTRO_NOK);
+ }
/* vertical motion vector range [-512, 511.75]
* (smaller for low levels) */
- if ((u32)(i32)(mv.ver+2048) >= (4096))
+ if ((u32)(i32)(mv.ver+2048) >= (4096)) {+ werrstr("vertical motion vector out of range");return(HANTRO_NOK);
+ }
}
tmp = h264bsdGetRefPicData(dpb, refIndex);
@@ -623,12 +627,16 @@
mv.ver += mvPred.ver;
/* horizontal motion vector range [-2048, 2047.75] */
- if ((u32)(i32)(mv.hor+8192) >= (16384))
+ if ((u32)(i32)(mv.hor+8192) >= (16384)) {+ werrstr("horizontal motion vector out of range");return(HANTRO_NOK);
+ }
/* vertical motion vector range [-512, 511.75] (smaller for low levels) */
- if ((u32)(i32)(mv.ver+2048) >= (4096))
+ if ((u32)(i32)(mv.ver+2048) >= (4096)) {+ werrstr("vertical motion vector out of range");return(HANTRO_NOK);
+ }
tmp = h264bsdGetRefPicData(dpb, refIndex);
if (tmp == NULL)
@@ -669,12 +677,16 @@
mv.ver += mvPred.ver;
/* horizontal motion vector range [-2048, 2047.75] */
- if ((u32)(i32)(mv.hor+8192) >= (16384))
+ if ((u32)(i32)(mv.hor+8192) >= (16384)) {+ werrstr("horizontal motion vector out of range");return(HANTRO_NOK);
+ }
/* vertical motion vector range [-512, 511.75] (smaller for low levels) */
- if ((u32)(i32)(mv.ver+2048) >= (4096))
+ if ((u32)(i32)(mv.ver+2048) >= (4096)) {+ werrstr("vertical motion vector out of range");return(HANTRO_NOK);
+ }
tmp = h264bsdGetRefPicData(dpb, refIndex);
if (tmp == NULL)
@@ -742,12 +754,16 @@
mv.ver += mvPred.ver;
/* horizontal motion vector range [-2048, 2047.75] */
- if ((u32)(i32)(mv.hor+8192) >= (16384))
+ if ((u32)(i32)(mv.hor+8192) >= (16384)) {+ werrstr("horizontal motion vector out of range");return(HANTRO_NOK);
+ }
/* vertical motion vector range [-512, 511.75] (smaller for low levels) */
- if ((u32)(i32)(mv.ver+2048) >= (4096))
+ if ((u32)(i32)(mv.ver+2048) >= (4096)) {+ werrstr("vertical motion vector out of range");return(HANTRO_NOK);
+ }
tmp = h264bsdGetRefPicData(dpb, refIndex);
if (tmp == NULL)
@@ -791,12 +807,16 @@
mv.ver += mvPred.ver;
/* horizontal motion vector range [-2048, 2047.75] */
- if ((u32)(i32)(mv.hor+8192) >= (16384))
+ if ((u32)(i32)(mv.hor+8192) >= (16384)) {+ werrstr("horizontal motion vector out of range");return(HANTRO_NOK);
+ }
/* vertical motion vector range [-512, 511.75] (smaller for low levels) */
- if ((u32)(i32)(mv.ver+2048) >= (4096))
+ if ((u32)(i32)(mv.ver+2048) >= (4096)) {+ werrstr("vertical motion vector out of range");return(HANTRO_NOK);
+ }
tmp = h264bsdGetRefPicData(dpb, refIndex);
if (tmp == NULL)
@@ -909,12 +929,16 @@
mv.ver += mvPred.ver;
/* horizontal motion vector range [-2048, 2047.75] */
- if (((u32)(i32)(mv.hor+8192) >= (16384)))
+ if (((u32)(i32)(mv.hor+8192) >= (16384))) {+ werrstr("horizontal motion vector out of range");return(HANTRO_NOK);
+ }
/* vertical motion vector range [-512, 511.75] (smaller for low levels) */
- if (((u32)(i32)(mv.ver+2048) >= (4096)))
+ if (((u32)(i32)(mv.ver+2048) >= (4096))) {+ werrstr("vertical motion vector out of range");return(HANTRO_NOK);
+ }
switch (subMbPartMode)
{--- a/src/h264bsd_nal_unit.c
+++ b/src/h264bsd_nal_unit.c
@@ -83,8 +83,10 @@
tmp = h264bsdGetBits(pStrmData, 1);
/* Assuming that NAL unit starts from byte boundary �> don't have to check
* following 7 bits for END_OF_STREAM */
- if (tmp == END_OF_STREAM)
+ if (tmp == END_OF_STREAM) {+ werrstr("end of stream");return(HANTRO_NOK);
+ }
tmp = h264bsdGetBits(pStrmData, 2);
pNalUnit->nalRefIdc = tmp;
--- a/src/h264bsd_pic_param_set.c
+++ b/src/h264bsd_pic_param_set.c
@@ -109,7 +109,7 @@
return(tmp);
if (pPicParamSet->picParameterSetId >= MAX_NUM_PIC_PARAM_SETS)
{- EPRINT("pic_parameter_set_id");+ werrstr("pic_parameter_set_id out of range: %d", pPicParamSet->picParameterSetId);return(HANTRO_NOK);
}
@@ -119,7 +119,7 @@
return(tmp);
if (pPicParamSet->seqParameterSetId >= MAX_NUM_SEQ_PARAM_SETS)
{- EPRINT("seq_param_set_id");+ werrstr("seq_param_set_id out of range: %d", pPicParamSet->seqParameterSetId);return(HANTRO_NOK);
}
@@ -127,7 +127,7 @@
tmp = h264bsdGetBits(pStrmData, 1);
if (tmp)
{- EPRINT("entropy_coding_mode_flag");+ werrstr("unexpected entropy_coding_mode_flag");return(HANTRO_NOK);
}
@@ -143,7 +143,7 @@
pPicParamSet->numSliceGroups = value + 1;
if (pPicParamSet->numSliceGroups > MAX_NUM_SLICE_GROUPS)
{- EPRINT("num_slice_groups_minus1");+ werrstr("num_slice_groups_minus1 out of range: %d", pPicParamSet->numSliceGroups);return(HANTRO_NOK);
}
@@ -156,7 +156,7 @@
return(tmp);
if (pPicParamSet->sliceGroupMapType > 6)
{- EPRINT("slice_group_map_type");+ werrstr("slice_group_map_type out of range: %d", pPicParamSet->sliceGroupMapType);return(HANTRO_NOK);
}
@@ -234,7 +234,7 @@
if ( pPicParamSet->sliceGroupId[i] >=
pPicParamSet->numSliceGroups )
{- EPRINT("slice_group_id");+ werrstr("slice_group_id out of range");return(HANTRO_NOK);
}
}
@@ -258,7 +258,7 @@
return(tmp);
if (value > 31)
{- EPRINT("num_ref_idx_l1_active_minus1");+ werrstr("num_ref_idx_l1_active_minus1 out of range: %d", value);return(HANTRO_NOK);
}
@@ -266,7 +266,7 @@
tmp = h264bsdGetBits(pStrmData, 1);
if (tmp)
{- EPRINT("weighted_pred_flag");+ werrstr("unexpected weighted_pred_flag");return(HANTRO_NOK);
}
@@ -274,7 +274,7 @@
tmp = h264bsdGetBits(pStrmData, 2);
if (tmp > 2)
{- EPRINT("weighted_bipred_idc");+ werrstr("weighted_bipred_idc out of range: %d", tmp);return(HANTRO_NOK);
}
@@ -284,7 +284,7 @@
return(tmp);
if ((itmp < -26) || (itmp > 25))
{- EPRINT("pic_init_qp_minus26");+ werrstr("pic_init_qp_minus26 out of range: %d", itmp);return(HANTRO_NOK);
}
pPicParamSet->picInitQp = (u32)(itmp + 26);
@@ -295,7 +295,7 @@
return(tmp);
if ((itmp < -26) || (itmp > 25))
{- EPRINT("pic_init_qs_minus26");+ werrstr("pic_init_qs_minus26 out of range: %d", itmp);return(HANTRO_NOK);
}
@@ -304,7 +304,7 @@
return(tmp);
if ((itmp < -12) || (itmp > 12))
{- EPRINT("chroma_qp_index_offset");+ werrstr("chroma_qp_index_offset out of range: %d", itmp);return(HANTRO_NOK);
}
pPicParamSet->chromaQpIndexOffset = itmp;
--- a/src/h264bsd_sei.c
+++ b/src/h264bsd_sei.c
@@ -418,8 +418,10 @@
&pBufferingPeriod->seqParameterSetId);
if (tmp != HANTRO_OK)
return(tmp);
- if (pBufferingPeriod->seqParameterSetId > 31)
+ if (pBufferingPeriod->seqParameterSetId > 31) {+ werrstr("seqParameterSetId out of range: %d", pBufferingPeriod->seqParameterSetId);return(HANTRO_NOK);
+ }
if (nalHrdBpPresentFlag)
{--- a/src/h264bsd_seq_param_set.c
+++ b/src/h264bsd_seq_param_set.c
@@ -99,9 +99,9 @@
tmp = h264bsdGetBits(pStrmData, 8);
if (tmp == END_OF_STREAM)
return(HANTRO_NOK);
- if (tmp != 66)
- {- DEBUG(("NOT BASELINE PROFILE %d\n", tmp));+ if (tmp != 66) {+ werrstr("not baseline profile: %d", tmp);+ return(HANTRO_NOK);
}
pSeqParamSet->profileIdc = tmp;
@@ -131,7 +131,7 @@
return(tmp);
if (pSeqParamSet->seqParameterSetId >= MAX_NUM_SEQ_PARAM_SETS)
{- EPRINT("seq_param_set_id");+ werrstr("seq_param_set_id out of range: %d", pSeqParamSet->seqParameterSetId);return(HANTRO_NOK);
}
@@ -141,7 +141,7 @@
return(tmp);
if (value > 12)
{- EPRINT("log2_max_frame_num_minus4");+ werrstr("log2_max_frame_num_minus4 out of range: %d", value);return(HANTRO_NOK);
}
/* maxFrameNum = 2^(log2_max_frame_num_minus4 + 4) */
@@ -153,7 +153,7 @@
return(tmp);
if (value > 2)
{- EPRINT("pic_order_cnt_type");+ werrstr("pic_order_cnt_type out of range: %d", value);return(HANTRO_NOK);
}
pSeqParamSet->picOrderCntType = value;
@@ -166,7 +166,7 @@
return(tmp);
if (value > 12)
{- EPRINT("log2_max_pic_order_cnt_lsb_minus4");+ werrstr("log2_max_pic_order_cnt_lsb_minus4 out of range: %d", value);return(HANTRO_NOK);
}
/* maxPicOrderCntLsb = 2^(log2_max_pic_order_cnt_lsb_minus4 + 4) */
@@ -196,7 +196,7 @@
return(tmp);
if (pSeqParamSet->numRefFramesInPicOrderCntCycle > 255)
{- EPRINT("num_ref_frames_in_pic_order_cnt_cycle");+ werrstr("num_ref_frames_in_pic_order_cnt_cycle out of range: %d", pSeqParamSet->numRefFramesInPicOrderCntCycle);return(HANTRO_NOK);
}
@@ -228,7 +228,7 @@
return(tmp);
if (pSeqParamSet->numRefFrames > MAX_NUM_REF_PICS)
{- EPRINT("num_ref_frames");+ werrstr("num_ref_frames out of range: %d", pSeqParamSet->numRefFrames);return(HANTRO_NOK);
}
@@ -254,7 +254,7 @@
return(HANTRO_NOK);
if (!tmp)
{- EPRINT("frame_mbs_only_flag");+ werrstr("frame_mbs_only_flag not set");return(HANTRO_NOK);
}
@@ -296,7 +296,7 @@
( 8 * (i32)pSeqParamSet->picHeightInMbs -
((i32)pSeqParamSet->frameCropBottomOffset + 1) ) ) )
{- EPRINT("frame_cropping");+ werrstr("frame_cropping invalid");return(HANTRO_NOK);
}
}
@@ -323,8 +323,10 @@
if (pSeqParamSet->vuiParametersPresentFlag)
{ALLOCATE(pSeqParamSet->vuiParameters, 1, vuiParameters_t);
- if (pSeqParamSet->vuiParameters == NULL)
+ if (pSeqParamSet->vuiParameters == NULL) {+ werrstr("no memory");return(MEMORY_ALLOCATION_ERROR);
+ }
tmp = h264bsdDecodeVuiParameters(pStrmData,
pSeqParamSet->vuiParameters);
if (tmp != HANTRO_OK)
--- a/src/h264bsd_vlc.c
+++ b/src/h264bsd_vlc.c
@@ -175,10 +175,12 @@
else if (bits == 1)
{*codeNum = BIG_CODE_NUM;
+ werrstr("big_code_num");return(HANTRO_NOK);
}
}
/* if more zeros than 32, it is an error */
+ werrstr("zeros > 32");return(HANTRO_NOK);
}
else
@@ -267,6 +269,8 @@
return(HANTRO_OK);
}
+ werrstr("h264bsdDecodeExpGolombSigned failed");+
return(HANTRO_NOK);
}
@@ -315,8 +319,10 @@
else
{/* range of valid codeNums [0,47] */
- if (codeNum > 47)
+ if (codeNum > 47) {+ werrstr("invalid codeNum %d", codeNum);return (HANTRO_NOK);
+ }
if (isIntra)
*value = codedBlockPatternIntra4x4[codeNum];
else