ref: b1d704f12af9b96b39ce1e1493c36bb4b3a3fb2a
parent: 220e4331bd4896e1cefabc713326ab28bc781f90
author: Cheng Chen <chengchen@google.com>
date: Thu Nov 5 10:26:54 EST 2020
Accumulate frame tpl stats and pass through rate control api Tpl stats is computed at the beginning of encoding the altref frame. We aggregate tpl stats of all blocks for every frame of the current group of picture. After the altref frame is encoded, the tpl stats is passed through the encode frame result to external environment. Change-Id: I2284f8cf9c45d35ba02f3ea45f0187edbbf48294
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -1025,6 +1025,7 @@
free_partition_info(cpi);
free_motion_vector_info(cpi);
free_fp_motion_vector_info(cpi);
+ free_tpl_stats_info(cpi);
#endif
vp9_free_ref_frame_buffers(cm->buffer_pool);
@@ -2665,6 +2666,7 @@
partition_info_init(cpi);
motion_vector_info_init(cpi);
fp_motion_vector_info_init(cpi);
+ tpl_stats_info_init(cpi);
#endif
return cpi;
@@ -5306,6 +5308,7 @@
#if CONFIG_RATE_CTRL
const PARTITION_INFO *partition_info,
const MOTION_VECTOR_INFO *motion_vector_info,
+ const TplDepStats *tpl_stats_info,
#endif // CONFIG_RATE_CTRL
ENCODE_FRAME_RESULT *encode_frame_result);
#endif // !CONFIG_REALTIME_ONLY
@@ -5520,7 +5523,7 @@
cpi->Source, coded_frame_buf, ref_frame_bufs, vp9_get_quantizer(cpi),
cm->bit_depth, cpi->oxcf.input_bit_depth, cpi->td.counts,
#if CONFIG_RATE_CTRL
- cpi->partition_info, cpi->motion_vector_info,
+ cpi->partition_info, cpi->motion_vector_info, cpi->tpl_stats_info,
#endif // CONFIG_RATE_CTRL
encode_frame_result);
}
@@ -7371,6 +7374,48 @@
}
}
+#if CONFIG_RATE_CTRL
+static void accumulate_frame_tpl_stats(VP9_COMP *cpi) {+ VP9_COMMON *const cm = &cpi->common;
+ const GF_GROUP *gf_group = &cpi->twopass.gf_group;
+ int show_frame_count = 0;
+ int frame_idx;
+ // Accumulate tpl stats for each frame in the current group of picture.
+ for (frame_idx = 1; frame_idx < gf_group->gf_group_size; ++frame_idx) {+ TplDepFrame *tpl_frame = &cpi->tpl_stats[frame_idx];
+ if (!tpl_frame->is_valid) continue;
+
+ TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
+ const int tpl_stride = tpl_frame->stride;
+ int64_t intra_cost_base = 0;
+ int64_t inter_cost_base = 0;
+ int64_t mc_dep_cost_base = 0;
+ int64_t mc_ref_cost_base = 0;
+ int64_t mc_flow_base = 0;
+ int row, col;
+
+ for (row = 0; row < cm->mi_rows && tpl_frame->is_valid; ++row) {+ for (col = 0; col < cm->mi_cols; ++col) {+ TplDepStats *this_stats = &tpl_stats[row * tpl_stride + col];
+ intra_cost_base += this_stats->intra_cost;
+ inter_cost_base += this_stats->inter_cost;
+ mc_dep_cost_base += this_stats->mc_dep_cost;
+ mc_ref_cost_base += this_stats->mc_ref_cost;
+ mc_flow_base += this_stats->mc_flow;
+ }
+ }
+
+ cpi->tpl_stats_info[show_frame_count].intra_cost = intra_cost_base;
+ cpi->tpl_stats_info[show_frame_count].inter_cost = inter_cost_base;
+ cpi->tpl_stats_info[show_frame_count].mc_dep_cost = mc_dep_cost_base;
+ cpi->tpl_stats_info[show_frame_count].mc_ref_cost = mc_ref_cost_base;
+ cpi->tpl_stats_info[show_frame_count].mc_flow = mc_flow_base;
+
+ ++show_frame_count;
+ }
+}
+#endif // CONFIG_RATE_CTRL
+
static void setup_tpl_stats(VP9_COMP *cpi) {GF_PICTURE gf_picture[MAX_ARF_GOP_SIZE];
const GF_GROUP *gf_group = &cpi->twopass.gf_group;
@@ -7393,6 +7438,10 @@
dump_tpl_stats(cpi, tpl_group_frames, gf_group, gf_picture, cpi->tpl_bsize);
#endif // DUMP_TPL_STATS
#endif // CONFIG_NON_GREEDY_MV
+
+#if CONFIG_RATE_CTRL
+ accumulate_frame_tpl_stats(cpi);
+#endif // CONFIG_RATE_CTRL
}
void vp9_get_ref_frame_info(FRAME_UPDATE_TYPE update_type, int ref_frame_flags,
@@ -7575,6 +7624,7 @@
#if CONFIG_RATE_CTRL
const PARTITION_INFO *partition_info,
const MOTION_VECTOR_INFO *motion_vector_info,
+ const TplDepStats *tpl_stats_info,
#endif // CONFIG_RATE_CTRL
ENCODE_FRAME_RESULT *encode_frame_result) {#if CONFIG_RATE_CTRL
@@ -7598,6 +7648,7 @@
copy_frame_counts(counts, &encode_frame_result->frame_counts);
encode_frame_result->partition_info = partition_info;
encode_frame_result->motion_vector_info = motion_vector_info;
+ encode_frame_result->tpl_stats_info = tpl_stats_info;
if (encode_frame_result->coded_frame.allocated) {yv12_buffer_to_image_buffer(&coded_frame_buf->buf,
&encode_frame_result->coded_frame);
--- a/vp9/encoder/vp9_encoder.h
+++ b/vp9/encoder/vp9_encoder.h
@@ -973,6 +973,7 @@
PARTITION_INFO *partition_info;
MOTION_VECTOR_INFO *motion_vector_info;
MOTION_VECTOR_INFO *fp_motion_vector_info;
+ TplDepStats *tpl_stats_info;
RATE_QSTEP_MODEL rq_model[ENCODE_FRAME_TYPES];
#endif
@@ -1029,6 +1030,23 @@
cpi->motion_vector_info = NULL;
}
+// Allocates memory for the tpl stats information.
+// Only called once in vp9_create_compressor().
+static INLINE void tpl_stats_info_init(struct VP9_COMP *cpi) {+ VP9_COMMON *const cm = &cpi->common;
+ CHECK_MEM_ERROR(
+ cm, cpi->tpl_stats_info,
+ (TplDepStats *)vpx_calloc(MAX_LAG_BUFFERS, sizeof(TplDepStats)));
+ memset(cpi->tpl_stats_info, 0, MAX_LAG_BUFFERS * sizeof(TplDepStats));
+}
+
+// Frees memory of the tpl stats information.
+// Only called once in dealloc_compressor_data().
+static INLINE void free_tpl_stats_info(struct VP9_COMP *cpi) {+ vpx_free(cpi->tpl_stats_info);
+ cpi->tpl_stats_info = NULL;
+}
+
// Allocates memory for the first pass motion vector information.
// The unit size is each 16x16 block.
// Only called once in vp9_create_compressor().
@@ -1091,6 +1109,7 @@
FRAME_COUNTS frame_counts;
const PARTITION_INFO *partition_info;
const MOTION_VECTOR_INFO *motion_vector_info;
+ const TplDepStats *tpl_stats_info;
IMAGE_BUFFER coded_frame;
RATE_QINDEX_HISTORY rq_history;
#endif // CONFIG_RATE_CTRL
--- a/vp9/simple_encode.cc
+++ b/vp9/simple_encode.cc
@@ -207,6 +207,24 @@
}
}
+static void update_tpl_stats_info(const TplDepStats *input_tpl_stats_info,
+ const int show_frame_count,
+ TplStatsInfo *output_tpl_stats_info) {+ int frame_idx;
+ for (frame_idx = 0; frame_idx < show_frame_count; ++frame_idx) {+ output_tpl_stats_info[frame_idx].intra_cost =
+ input_tpl_stats_info[frame_idx].intra_cost;
+ output_tpl_stats_info[frame_idx].inter_cost =
+ input_tpl_stats_info[frame_idx].inter_cost;
+ output_tpl_stats_info[frame_idx].mc_flow =
+ input_tpl_stats_info[frame_idx].mc_flow;
+ output_tpl_stats_info[frame_idx].mc_dep_cost =
+ input_tpl_stats_info[frame_idx].mc_dep_cost;
+ output_tpl_stats_info[frame_idx].mc_ref_cost =
+ input_tpl_stats_info[frame_idx].mc_ref_cost;
+ }
+}
+
static void update_frame_counts(const FRAME_COUNTS *input_counts,
FrameCounts *output_counts) {// Init array sizes.
@@ -486,6 +504,7 @@
encode_frame_result->num_cols_4x4);
encode_frame_result->motion_vector_info.resize(
encode_frame_result->num_rows_4x4 * encode_frame_result->num_cols_4x4);
+ encode_frame_result->tpl_stats_info.resize(MAX_LAG_BUFFERS);
if (encode_frame_result->coding_data.get() == nullptr) {return false;
@@ -507,7 +526,7 @@
}
static void update_encode_frame_result(
- EncodeFrameResult *encode_frame_result,
+ EncodeFrameResult *encode_frame_result, const int show_frame_count,
const ENCODE_FRAME_RESULT *encode_frame_info) {encode_frame_result->coding_data_bit_size =
encode_frame_result->coding_data_byte_size * 8;
@@ -536,6 +555,10 @@
kMotionVectorSubPixelPrecision);
update_frame_counts(&encode_frame_info->frame_counts,
&encode_frame_result->frame_counts);
+ if (encode_frame_result->frame_type == kFrameTypeAltRef) {+ update_tpl_stats_info(encode_frame_info->tpl_stats_info, show_frame_count,
+ &encode_frame_result->tpl_stats_info[0]);
+ }
encode_frame_result_update_rq_history(&encode_frame_info->rq_history,
encode_frame_result);
}
@@ -1169,7 +1192,10 @@
abort();
}
- update_encode_frame_result(encode_frame_result, &encode_frame_info);
+ const GroupOfPicture group_of_picture = this->ObserveGroupOfPicture();
+ const int show_frame_count = group_of_picture.show_frame_count;
+ update_encode_frame_result(encode_frame_result, show_frame_count,
+ &encode_frame_info);
PostUpdateState(*encode_frame_result);
} else {// TODO(angiebird): Clean up encode_frame_result.
--- a/vp9/simple_encode.h
+++ b/vp9/simple_encode.h
@@ -87,6 +87,24 @@
double mv_column[2];
};
+// Accumulated tpl stats of all blocks in one frame.
+// For each frame, the tpl stats are computed per 32x32 block.
+struct TplStatsInfo {+ // Intra complexity: the sum of absolute transform difference (SATD) of
+ // intra predicted residuals.
+ int64_t intra_cost;
+ // Inter complexity: the SATD of inter predicted residuals.
+ int64_t inter_cost;
+ // Motion compensated information flow. It measures how much information
+ // is propagated from the current frame to other frames.
+ int64_t mc_flow;
+ // Motion compensated dependency cost. It equals to its own intra_cost
+ // plus the mc_flow.
+ int64_t mc_dep_cost;
+ // Motion compensated reference cost.
+ int64_t mc_ref_cost;
+};
+
struct RefFrameInfo {int coding_indexes[kRefFrameTypeMax];
@@ -261,6 +279,18 @@
// Similar to partition info, all 4x4 blocks inside the same partition block
// share the same motion vector information.
std::vector<MotionVectorInfo> motion_vector_info;
+ // A vector of the tpl stats information.
+ // The tpl stats measure the complexity of a frame, as well as the
+ // informatioin propagated along the motion trajactory between frames, in
+ // the reference frame structure.
+ // The tpl stats could be used as a more accurate spatial and temporal
+ // complexity measure in addition to the first pass stats.
+ // The vector contains tpl stats for all show frames in a GOP.
+ // The tpl stats stored in the vector is according to the encoding order.
+ // For example, suppose there are N show frames for the current GOP.
+ // Then tpl_stats_info[0] stores the information of the first frame to be
+ // encoded for this GOP, i.e, the AltRef frame.
+ std::vector<TplStatsInfo> tpl_stats_info;
ImageBuffer coded_frame;
// recode_count, q_index_history and rate_history are only available when