ref: b6b40d30158171a14200353590861c5df7b727b9
parent: c8fa9e9d53fb69aa34135dcbde59368beb03ceb5
parent: 00a39d6e046c65c0ac7c57926e892b7e10afda34
author: James Zern <jzern@google.com>
date: Thu Feb 18 13:53:13 EST 2016
Merge changes from topic 'vp9-missing-alloc-checks' * changes: vp9_svc_layercontext: add missing alloc checks vp9_resize: add missing alloc checks vp9_encoder: add missing alloc checks vp9/decoder,resize_mv_buffer: add missing alloc check vp9_cyclic_refresh_alloc: correct cleanup on error
--- a/vp9/decoder/vp9_decodeframe.c
+++ b/vp9/decoder/vp9_decodeframe.c
@@ -1221,8 +1221,9 @@
vpx_free(cm->cur_frame->mvs);
cm->cur_frame->mi_rows = cm->mi_rows;
cm->cur_frame->mi_cols = cm->mi_cols;
- cm->cur_frame->mvs = (MV_REF *)vpx_calloc(cm->mi_rows * cm->mi_cols,
- sizeof(*cm->cur_frame->mvs));
+ CHECK_MEM_ERROR(cm, cm->cur_frame->mvs,
+ (MV_REF *)vpx_calloc(cm->mi_rows * cm->mi_cols,
+ sizeof(*cm->cur_frame->mvs)));
}
static void resize_context_buffers(VP9_COMMON *cm, int width, int height) {--- a/vp9/encoder/vp9_aq_cyclicrefresh.c
+++ b/vp9/encoder/vp9_aq_cyclicrefresh.c
@@ -30,13 +30,13 @@
cr->map = vpx_calloc(mi_rows * mi_cols, sizeof(*cr->map));
if (cr->map == NULL) {- vpx_free(cr);
+ vp9_cyclic_refresh_free(cr);
return NULL;
}
last_coded_q_map_size = mi_rows * mi_cols * sizeof(*cr->last_coded_q_map);
cr->last_coded_q_map = vpx_malloc(last_coded_q_map_size);
if (cr->last_coded_q_map == NULL) {- vpx_free(cr);
+ vp9_cyclic_refresh_free(cr);
return NULL;
}
assert(MAXQ <= 255);
@@ -45,7 +45,7 @@
consec_zero_mv_size = mi_rows * mi_cols * sizeof(*cr->consec_zero_mv);
cr->consec_zero_mv = vpx_malloc(consec_zero_mv_size);
if (cr->consec_zero_mv == NULL) {- vpx_free(cr);
+ vp9_cyclic_refresh_free(cr);
return NULL;
}
memset(cr->consec_zero_mv, 0, consec_zero_mv_size);
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -1771,8 +1771,9 @@
}
if (cpi->b_calculate_consistency) {- cpi->ssim_vars = vpx_malloc(sizeof(*cpi->ssim_vars) *
- 4 * cpi->common.mi_rows * cpi->common.mi_cols);
+ CHECK_MEM_ERROR(cm, cpi->ssim_vars,
+ vpx_malloc(sizeof(*cpi->ssim_vars) * 4 *
+ cpi->common.mi_rows * cpi->common.mi_cols));
cpi->worst_consistency = 100.0;
}
@@ -2899,7 +2900,7 @@
vpx_extend_frame_inner_borders(cm->frame_to_show);
}
-static INLINE void alloc_frame_mvs(const VP9_COMMON *cm,
+static INLINE void alloc_frame_mvs(VP9_COMMON *const cm,
int buffer_idx) {RefCntBuffer *const new_fb_ptr = &cm->buffer_pool->frame_bufs[buffer_idx];
if (new_fb_ptr->mvs == NULL ||
@@ -2906,9 +2907,9 @@
new_fb_ptr->mi_rows < cm->mi_rows ||
new_fb_ptr->mi_cols < cm->mi_cols) {vpx_free(new_fb_ptr->mvs);
- new_fb_ptr->mvs =
- (MV_REF *)vpx_calloc(cm->mi_rows * cm->mi_cols,
- sizeof(*new_fb_ptr->mvs));
+ CHECK_MEM_ERROR(cm, new_fb_ptr->mvs,
+ (MV_REF *)vpx_calloc(cm->mi_rows * cm->mi_cols,
+ sizeof(*new_fb_ptr->mvs)));
new_fb_ptr->mi_rows = cm->mi_rows;
new_fb_ptr->mi_cols = cm->mi_cols;
}
@@ -2946,12 +2947,13 @@
if (force_scaling ||
new_fb_ptr->buf.y_crop_width != cm->width ||
new_fb_ptr->buf.y_crop_height != cm->height) {- vpx_realloc_frame_buffer(&new_fb_ptr->buf,
- cm->width, cm->height,
- cm->subsampling_x, cm->subsampling_y,
- cm->use_highbitdepth,
- VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
- NULL, NULL, NULL);
+ if (vpx_realloc_frame_buffer(&new_fb_ptr->buf, cm->width, cm->height,
+ cm->subsampling_x, cm->subsampling_y,
+ cm->use_highbitdepth,
+ VP9_ENC_BORDER_IN_PIXELS,
+ cm->byte_alignment, NULL, NULL, NULL))
+ vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
+ "Failed to allocate frame buffer");
scale_and_extend_frame(ref, &new_fb_ptr->buf, (int)cm->bit_depth);
cpi->scaled_ref_idx[ref_frame - 1] = new_fb;
alloc_frame_mvs(cm, new_fb);
@@ -2971,11 +2973,12 @@
if (force_scaling ||
new_fb_ptr->buf.y_crop_width != cm->width ||
new_fb_ptr->buf.y_crop_height != cm->height) {- vpx_realloc_frame_buffer(&new_fb_ptr->buf,
- cm->width, cm->height,
- cm->subsampling_x, cm->subsampling_y,
- VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
- NULL, NULL, NULL);
+ if (vpx_realloc_frame_buffer(&new_fb_ptr->buf, cm->width, cm->height,
+ cm->subsampling_x, cm->subsampling_y,
+ VP9_ENC_BORDER_IN_PIXELS,
+ cm->byte_alignment, NULL, NULL, NULL))
+ vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
+ "Failed to allocate frame buffer");
vp9_scale_and_extend_frame(ref, &new_fb_ptr->buf);
cpi->scaled_ref_idx[ref_frame - 1] = new_fb;
alloc_frame_mvs(cm, new_fb);
@@ -3222,12 +3225,14 @@
VP9_COMMON *const cm = &cpi->common;
if (cpi->oxcf.noise_sensitivity > 0 &&
!cpi->denoiser.frame_buffer_initialized) {- vp9_denoiser_alloc(&(cpi->denoiser), cm->width, cm->height,
- cm->subsampling_x, cm->subsampling_y,
+ if (vp9_denoiser_alloc(&cpi->denoiser, cm->width, cm->height,
+ cm->subsampling_x, cm->subsampling_y,
#if CONFIG_VP9_HIGHBITDEPTH
- cm->use_highbitdepth,
+ cm->use_highbitdepth,
#endif
- VP9_ENC_BORDER_IN_PIXELS);
+ VP9_ENC_BORDER_IN_PIXELS))
+ vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
+ "Failed to allocate denoiser");
}
}
#endif
@@ -3300,14 +3305,15 @@
alloc_frame_mvs(cm, cm->new_fb_idx);
// Reset the frame pointers to the current frame size.
- vpx_realloc_frame_buffer(get_frame_new_buffer(cm),
- cm->width, cm->height,
- cm->subsampling_x, cm->subsampling_y,
+ if (vpx_realloc_frame_buffer(get_frame_new_buffer(cm), cm->width, cm->height,
+ cm->subsampling_x, cm->subsampling_y,
#if CONFIG_VP9_HIGHBITDEPTH
- cm->use_highbitdepth,
+ cm->use_highbitdepth,
#endif
- VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
- NULL, NULL, NULL);
+ VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
+ NULL, NULL, NULL))
+ vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
+ "Failed to allocate frame buffer");
alloc_util_frame_buffers(cpi);
init_motion_estimation(cpi);
--- a/vp9/encoder/vp9_resize.c
+++ b/vp9/encoder/vp9_resize.c
@@ -462,6 +462,7 @@
int filteredlength = length;
if (!tmpbuf) {tmpbuf = (uint8_t *)malloc(sizeof(uint8_t) * length);
+ if (tmpbuf == NULL) return;
otmp = tmpbuf;
} else {otmp = buf;
@@ -521,6 +522,7 @@
uint8_t *tmpbuf = (uint8_t *)malloc(sizeof(uint8_t) *
(width < height ? height : width));
uint8_t *arrbuf = (uint8_t *)malloc(sizeof(uint8_t) * (height + height2));
+ if (intbuf == NULL || tmpbuf == NULL || arrbuf == NULL) goto Error;
assert(width > 0);
assert(height > 0);
assert(width2 > 0);
@@ -533,6 +535,8 @@
resize_multistep(arrbuf, height, arrbuf + height, height2, tmpbuf);
fill_arr_to_col(output + i, out_stride, height2, arrbuf + height);
}
+
+ Error:
free(intbuf);
free(tmpbuf);
free(arrbuf);
@@ -755,6 +759,7 @@
int filteredlength = length;
if (!tmpbuf) {tmpbuf = (uint16_t *)malloc(sizeof(uint16_t) * length);
+ if (tmpbuf == NULL) return;
otmp = tmpbuf;
} else {otmp = buf;
@@ -817,6 +822,7 @@
uint16_t *tmpbuf = (uint16_t *)malloc(sizeof(uint16_t) *
(width < height ? height : width));
uint16_t *arrbuf = (uint16_t *)malloc(sizeof(uint16_t) * (height + height2));
+ if (intbuf == NULL || tmpbuf == NULL || arrbuf == NULL) goto Error;
for (i = 0; i < height; ++i) {highbd_resize_multistep(CONVERT_TO_SHORTPTR(input + in_stride * i), width,
intbuf + width2 * i, width2, tmpbuf, bd);
@@ -828,6 +834,8 @@
highbd_fill_arr_to_col(CONVERT_TO_SHORTPTR(output + i), out_stride, height2,
arrbuf + height);
}
+
+ Error:
free(intbuf);
free(tmpbuf);
free(arrbuf);
--- a/vp9/encoder/vp9_svc_layercontext.c
+++ b/vp9/encoder/vp9_svc_layercontext.c
@@ -118,15 +118,20 @@
tl == 0) {size_t last_coded_q_map_size;
size_t consec_zero_mv_size;
+ VP9_COMMON *const cm = &cpi->common;
lc->sb_index = 0;
- lc->map = vpx_malloc(mi_rows * mi_cols * sizeof(signed char));
+ CHECK_MEM_ERROR(cm, lc->map,
+ vpx_malloc(mi_rows * mi_cols * sizeof(*lc->map)));
memset(lc->map, 0, mi_rows * mi_cols);
- last_coded_q_map_size = mi_rows * mi_cols * sizeof(uint8_t);
- lc->last_coded_q_map = vpx_malloc(last_coded_q_map_size);
+ last_coded_q_map_size = mi_rows * mi_cols *
+ sizeof(*lc->last_coded_q_map);
+ CHECK_MEM_ERROR(cm, lc->last_coded_q_map,
+ vpx_malloc(last_coded_q_map_size));
assert(MAXQ <= 255);
memset(lc->last_coded_q_map, MAXQ, last_coded_q_map_size);
- consec_zero_mv_size = mi_rows * mi_cols * sizeof(uint8_t);
- lc->consec_zero_mv = vpx_malloc(consec_zero_mv_size);
+ consec_zero_mv_size = mi_rows * mi_cols * sizeof(*lc->consec_zero_mv);
+ CHECK_MEM_ERROR(cm, lc->consec_zero_mv,
+ vpx_malloc(consec_zero_mv_size));
memset(lc->consec_zero_mv, 0, consec_zero_mv_size);
}
}
--
⑨