shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix usage of value returned by vp8_pick_intra4x4mby_modes

The value of distortion2 returned by vp8_pick_intra4x4mby_modes
was being overwritten by the value returned by get16x16prederror
before it was tested.

Change-Id: If00e80332b272c5545c3a7e381c8041e8319b41a

--- a/vp8/encoder/pickinter.c

+++ b/vp8/encoder/pickinter.c

@@ -639,10 +639,10 @@

         switch (this_mode)

         {

         case B_PRED:

-            distortion2 = *returndistortion;                    // Best so far passed in as breakout value to vp8_pick_intra4x4mby_modes

-            vp8_pick_intra4x4mby_modes(IF_RTCD(&cpi->rtcd), x, &rate, &distortion2);

-            rate2 += rate;

-            distortion2 = VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16prederror)(x->src.y_buffer, x->src.y_stride, x->e_mbd.predictor, 16, 0x7fffffff);

+            // Pass best so far to vp8_pick_intra4x4mby_modes to use as breakout

+            distortion2 = *returndistortion;

+            vp8_pick_intra4x4mby_modes(IF_RTCD(&cpi->rtcd), x,

+                                         &rate, &distortion2);

             if (distortion2 == INT_MAX)

             {

@@ -650,6 +650,11 @@

             }

             else

             {

+                rate2 += rate;

+                distortion2 = VARIANCE_INVOKE

+                                (&cpi->rtcd.variance, get16x16prederror)(

+                                    x->src.y_buffer, x->src.y_stride,

+                                    x->e_mbd.predictor, 16, 0x7fffffff);

                 this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);

                 if (this_rd < best_intra_rd)

-- 

⑨