shithub: libvpx

Info  •  Files  •  Log  •  Branches

Merge "Rationalize type to avoid integer out of range"

--- a/vp9/encoder/vp9_mcomp.c

+++ b/vp9/encoder/vp9_mcomp.c

@@ -8,6 +8,7 @@

  *  be found in the AUTHORS file in the root of the source tree.

  */

+#include <assert.h>

 #include <limits.h>

 #include <math.h>

 #include <stdio.h>

@@ -69,6 +70,8 @@

 static INLINE int mv_cost(const MV *mv,

                           const int *joint_cost, int *const comp_cost[2]) {

+  assert(mv->row >= -MV_MAX && mv->row < MV_MAX);

+  assert(mv->col >= -MV_MAX && mv->col < MV_MAX);

   return joint_cost[vp9_get_mv_joint(mv)] +

              comp_cost[0][mv->row] + comp_cost[1][mv->col];

 }

@@ -445,7 +448,7 @@

   if ((abs(bestmv->col - ref_mv->col) > (MAX_FULL_PEL_VAL << 3)) ||

       (abs(bestmv->row - ref_mv->row) > (MAX_FULL_PEL_VAL << 3)))

-    return INT_MAX;

+    return UINT32_MAX;

   return besterr;

 }

@@ -599,7 +602,7 @@

   if ((abs(bestmv->col - ref_mv->col) > (MAX_FULL_PEL_VAL << 3)) ||

       (abs(bestmv->row - ref_mv->row) > (MAX_FULL_PEL_VAL << 3)))

-    return INT_MAX;

+    return UINT32_MAX;

   return besterr;

 }

--- a/vp9/encoder/vp9_rdopt.c

+++ b/vp9/encoder/vp9_rdopt.c

@@ -1594,7 +1594,7 @@

   // Do joint motion search in compound mode to get more accurate mv.

   struct buf_2d backup_yv12[2][MAX_MB_PLANE];

-  int last_besterr[2] = {INT_MAX, INT_MAX};

+  uint32_t last_besterr[2] = {UINT32_MAX, UINT32_MAX};

   const YV12_BUFFER_CONFIG *const scaled_ref_frame[2] = {

     vp9_get_scaled_ref_frame(cpi, mi->ref_frame[0]),

     vp9_get_scaled_ref_frame(cpi, mi->ref_frame[1])

@@ -1640,7 +1640,7 @@

   // and break out of the search loop if it couldn't find a better mv.

   for (ite = 0; ite < 4; ite++) {

     struct buf_2d ref_yv12[2];

-    int bestsme = INT_MAX;

+    uint32_t bestsme = UINT32_MAX;

     int sadpb = x->sadperbit16;

     MV tmp_mv;

     int search_range = 3;

@@ -1705,7 +1705,7 @@

                                        search_range,

                                        &cpi->fn_ptr[bsize],

                                        &ref_mv[id].as_mv, second_pred);

-    if (bestsme < INT_MAX)

+    if (bestsme < UINT32_MAX)

       bestsme = vp9_get_mvpred_av_var(x, &tmp_mv, &ref_mv[id].as_mv,

                                       second_pred, &cpi->fn_ptr[bsize], 1);

@@ -1714,7 +1714,7 @@

     x->mv_row_min = tmp_row_min;

     x->mv_row_max = tmp_row_max;

-    if (bestsme < INT_MAX) {

+    if (bestsme < UINT32_MAX) {

       uint32_t dis; /* TODO: use dis in distortion calculation later. */

       uint32_t sse;

       bestsme = cpi->find_fractional_mv_step(

@@ -1860,7 +1860,7 @@

             seg_mvs[i][mi->ref_frame[0]].as_int == INVALID_MV) {

           MV *const new_mv = &mode_mv[NEWMV][0].as_mv;

           int step_param = 0;

-          int bestsme = INT_MAX;

+          uint32_t bestsme = UINT32_MAX;

           int sadpb = x->sadperbit4;

           MV mvp_full;

           int max_mv;

@@ -1915,7 +1915,7 @@

               &bsi->ref_mv[0]->as_mv, new_mv,

               INT_MAX, 1);

-          if (bestsme < INT_MAX) {

+          if (bestsme < UINT32_MAX) {

             uint32_t distortion;

             cpi->find_fractional_mv_step(

                 x,

-- 

⑨