shithub: dav1d

Info  •  Files  •  Log  •  Branches

Deal with chroma coefficients that are exactly 0x100000

For chroma coefficients that are masked (&= 0xfffff) to no value, the
context becomes in a weird state where it has no magnitude (ctx & 0x3f
== 0), but it does have a sign (ctx & 0xc0 != 0x40). Our old code
checked just the magnitude part of the context to set the skip context
of neighbouring blocks, but libaom uses both sign and magnitude for
this purpose. Therefore, adjust our code so it does the same thing.

Luma code only checks magnitude for this purpose and is thus not
affected by this peculiarity. Fixes #325.

--- a/src/recon_tmpl.c

+++ b/src/recon_tmpl.c

@@ -70,10 +70,10 @@

         const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444;

         const int not_one_blk = b_dim[2] - (!!b_dim[2] && ss_hor) > t_dim->lw ||

                                 b_dim[3] - (!!b_dim[3] && ss_ver) > t_dim->lh;

-        int ca, cl;

+        unsigned ca, cl;

-#define MERGE_CTX(dir, type, mask) \

-        c##dir = !!((*(const type *) dir) & mask); \

+#define MERGE_CTX(dir, type, no_val) \

+        c##dir = *(const type *) dir != no_val; \

         break

         switch (t_dim->lw) {

@@ -83,17 +83,17 @@

          * and will therefore complain about the use of uninitialized variables

          * when compiled in debug mode if we put the default case at the end. */

         default: assert(0); /* fall-through */

-        case TX_4X4:   MERGE_CTX(a, uint8_t,  0x3F);

-        case TX_8X8:   MERGE_CTX(a, uint16_t, 0x3F3F);

-        case TX_16X16: MERGE_CTX(a, uint32_t, 0x3F3F3F3FU);

-        case TX_32X32: MERGE_CTX(a, uint64_t, 0x3F3F3F3F3F3F3F3FULL);

+        case TX_4X4:   MERGE_CTX(a, uint8_t,  0x40);

+        case TX_8X8:   MERGE_CTX(a, uint16_t, 0x4040);

+        case TX_16X16: MERGE_CTX(a, uint32_t, 0x40404040U);

+        case TX_32X32: MERGE_CTX(a, uint64_t, 0x4040404040404040ULL);

         }

         switch (t_dim->lh) {

         default: assert(0); /* fall-through */

-        case TX_4X4:   MERGE_CTX(l, uint8_t,  0x3F);

-        case TX_8X8:   MERGE_CTX(l, uint16_t, 0x3F3F);

-        case TX_16X16: MERGE_CTX(l, uint32_t, 0x3F3F3F3FU);

-        case TX_32X32: MERGE_CTX(l, uint64_t, 0x3F3F3F3F3F3F3F3FULL);

+        case TX_4X4:   MERGE_CTX(l, uint8_t,  0x40);

+        case TX_8X8:   MERGE_CTX(l, uint16_t, 0x4040);

+        case TX_16X16: MERGE_CTX(l, uint32_t, 0x40404040U);

+        case TX_32X32: MERGE_CTX(l, uint64_t, 0x4040404040404040ULL);

         }

 #undef MERGE_CTX