shithub: libvpx

Download patch

ref: 18303b1263170e33369a2925b191844ea0ff0b05
parent: ef0ca2deaa27f3f88242741e3e0c4af4179eab69
author: Tero Rintaluoma <teror@google.com>
date: Fri Jul 5 09:53:36 EDT 2013 

Fix intermediate height in convolve

intermediate_height for horizontal filtering must be at least 8
pixels to be able to do vertical filtering correctly. Currently
it can be less for small block and y_step_q4 sizes.

Change-Id: I2ee28b0591b2041c2fa9844d0ae2ff8a1a59cc21


--- a/vp9/common/vp9_convolve.c
+++ b/vp9/common/vp9_convolve.c
@@ -217,12 +217,13 @@
    * h == 64, taps == 8.
    */
   uint8_t temp[64 * 135];
-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;
 
   assert(w <= 64);
   assert(h <= 64);
   assert(taps <= 8);
   assert(y_step_q4 <= 32);
+  assert(x_step_q4 <= 32);
 
   if (intermediate_height < h)
     intermediate_height = h;
@@ -246,12 +247,13 @@
    * h == 64, taps == 8.
    */
   uint8_t temp[64 * 135];
-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;
 
   assert(w <= 64);
   assert(h <= 64);
   assert(taps <= 8);
   assert(y_step_q4 <= 32);
+  assert(x_step_q4 <= 32);
 
   if (intermediate_height < h)
     intermediate_height = h;
-- 
⑨