shithub: libvpx

Info  •  Files  •  Log  •  Branches

vpx_dsp/inv_txfm: fix int sanitizer warnings

with vp9-highbitdepth off.
Unit Test:  SSE2/Trans16x16DCT , VP9/LevelTest.TestTargetLevel20Large, VP9/CpuSpeedTest

implicit conversion from type 'int32_t' (aka 'int') of value -32851
(32-bit, signed) to type 'tran_low_t' (aka 'short') changed the value to
32685 (16-bit, signed)

BUG=webm:1615
BUG=webm:1647

Change-Id: I9ef064dc9ac734379628565ff6505b0876984123

--- a/vpx_dsp/inv_txfm.c

+++ b/vpx_dsp/inv_txfm.c

@@ -701,22 +701,22 @@

   step2[15] = step1[15];

   // stage 7

-  output[0] = WRAPLOW(step2[0] + step2[15]);

-  output[1] = WRAPLOW(step2[1] + step2[14]);

-  output[2] = WRAPLOW(step2[2] + step2[13]);

-  output[3] = WRAPLOW(step2[3] + step2[12]);

-  output[4] = WRAPLOW(step2[4] + step2[11]);

-  output[5] = WRAPLOW(step2[5] + step2[10]);

-  output[6] = WRAPLOW(step2[6] + step2[9]);

-  output[7] = WRAPLOW(step2[7] + step2[8]);

-  output[8] = WRAPLOW(step2[7] - step2[8]);

-  output[9] = WRAPLOW(step2[6] - step2[9]);

-  output[10] = WRAPLOW(step2[5] - step2[10]);

-  output[11] = WRAPLOW(step2[4] - step2[11]);

-  output[12] = WRAPLOW(step2[3] - step2[12]);

-  output[13] = WRAPLOW(step2[2] - step2[13]);

-  output[14] = WRAPLOW(step2[1] - step2[14]);

-  output[15] = WRAPLOW(step2[0] - step2[15]);

+  output[0] = (tran_low_t)WRAPLOW(step2[0] + step2[15]);

+  output[1] = (tran_low_t)WRAPLOW(step2[1] + step2[14]);

+  output[2] = (tran_low_t)WRAPLOW(step2[2] + step2[13]);

+  output[3] = (tran_low_t)WRAPLOW(step2[3] + step2[12]);

+  output[4] = (tran_low_t)WRAPLOW(step2[4] + step2[11]);

+  output[5] = (tran_low_t)WRAPLOW(step2[5] + step2[10]);

+  output[6] = (tran_low_t)WRAPLOW(step2[6] + step2[9]);

+  output[7] = (tran_low_t)WRAPLOW(step2[7] + step2[8]);

+  output[8] = (tran_low_t)WRAPLOW(step2[7] - step2[8]);

+  output[9] = (tran_low_t)WRAPLOW(step2[6] - step2[9]);

+  output[10] = (tran_low_t)WRAPLOW(step2[5] - step2[10]);

+  output[11] = (tran_low_t)WRAPLOW(step2[4] - step2[11]);

+  output[12] = (tran_low_t)WRAPLOW(step2[3] - step2[12]);

+  output[13] = (tran_low_t)WRAPLOW(step2[2] - step2[13]);

+  output[14] = (tran_low_t)WRAPLOW(step2[1] - step2[14]);

+  output[15] = (tran_low_t)WRAPLOW(step2[0] - step2[15]);

 }

 void vpx_idct16x16_256_add_c(const tran_low_t *input, uint8_t *dest,