lavfi/drawutils: overhaul to improve pixel format support

- No longer mixes u8 and u16 component accesses (this was UB)
- De-duplicated 8->16 conversion
- De-duplicated component -> plane+offset conversion
- De-duplicated planar + packed RGB
- No longer calls ff_fill_rgba_map
- Removed redundant comp_mask data member
- RGB0 and related formats no longer write an alpha value to the 0 byte
- Non-planar YA formats now work correctly
- High-bit-depth semi-planar YUV now works correctly

2 files changed, 48 insertions, 63 deletions

diff --git a/libavfilter/drawutils.c b/libavfilter/drawutils.c
index 5308fcbc0f..cbb2582fe0 100644
--- a/libavfilter/drawutils.c
+++ b/libavfilter/drawutils.c
@@ -137,66 +137,49 @@ int ff_draw_init(FFDrawContext *draw, enum AVPixelFormat format, unsigned flags)
     memcpy(draw->pixelstep, pixelstep, sizeof(draw->pixelstep));
     draw->hsub[1] = draw->hsub[2] = draw->hsub_max = desc->log2_chroma_w;
     draw->vsub[1] = draw->vsub[2] = draw->vsub_max = desc->log2_chroma_h;
-    for (i = 0; i < (desc->nb_components - !!(desc->flags & AV_PIX_FMT_FLAG_ALPHA && !(flags & FF_DRAW_PROCESS_ALPHA))); i++)
-        draw->comp_mask[desc->comp[i].plane] |=
-            1 << desc->comp[i].offset;
     return 0;
 }
 
 void ff_draw_color(FFDrawContext *draw, FFDrawColor *color, const uint8_t rgba[4])
 {
     unsigned i;
-    uint8_t rgba_map[4];
+    uint8_t tmp8[4];
+    const AVPixFmtDescriptor *desc = draw->desc;
 
     if (rgba != color->rgba)
         memcpy(color->rgba, rgba, sizeof(color->rgba));
-    if ((draw->desc->flags & AV_PIX_FMT_FLAG_RGB) &&
-        ff_fill_rgba_map(rgba_map, draw->format) >= 0) {
-        if (draw->nb_planes == 1) {
-            for (i = 0; i < 4; i++) {
-                color->comp[0].u8[rgba_map[i]] = rgba[i];
-                if (draw->desc->comp[rgba_map[i]].depth > 8) {
-                    color->comp[0].u16[rgba_map[i]] = color->comp[0].u8[rgba_map[i]] << 8;
-                }
-            }
-        } else {
-            for (i = 0; i < 4; i++) {
-                color->comp[rgba_map[i]].u8[0] = rgba[i];
-                if (draw->desc->comp[rgba_map[i]].depth > 8)
-                    color->comp[rgba_map[i]].u16[0] = color->comp[rgba_map[i]].u8[0] << (draw->desc->comp[rgba_map[i]].depth - 8);
-            }
-        }
+
+    memset(color->comp, 0, sizeof(color->comp));
+
+    if (draw->desc->flags & AV_PIX_FMT_FLAG_RGB) {
+        memcpy(tmp8, rgba, sizeof(tmp8));
     } else if (draw->nb_planes >= 2) {
         /* assume YUV */
-        const AVPixFmtDescriptor *desc = draw->desc;
-        color->comp[desc->comp[0].plane].u8[desc->comp[0].offset] = draw->full_range ? RGB_TO_Y_JPEG(rgba[0], rgba[1], rgba[2]) : RGB_TO_Y_CCIR(rgba[0], rgba[1], rgba[2]);
-        color->comp[desc->comp[1].plane].u8[desc->comp[1].offset] = draw->full_range ? RGB_TO_U_JPEG(rgba[0], rgba[1], rgba[2]) : RGB_TO_U_CCIR(rgba[0], rgba[1], rgba[2], 0);
-        color->comp[desc->comp[2].plane].u8[desc->comp[2].offset] = draw->full_range ? RGB_TO_V_JPEG(rgba[0], rgba[1], rgba[2]) : RGB_TO_V_CCIR(rgba[0], rgba[1], rgba[2], 0);
-        color->comp[3].u8[0] = rgba[3];
-#define EXPAND(compn) \
-        if (desc->comp[compn].depth > 8) \
-            color->comp[desc->comp[compn].plane].u16[desc->comp[compn].offset] = \
-            color->comp[desc->comp[compn].plane].u8[desc->comp[compn].offset] << \
-                (draw->desc->comp[compn].depth + draw->desc->comp[compn].shift - 8)
-        EXPAND(3);
-        EXPAND(2);
-        EXPAND(1);
-        EXPAND(0);
+        tmp8[0] = draw->full_range ? RGB_TO_Y_JPEG(rgba[0], rgba[1], rgba[2]) : RGB_TO_Y_CCIR(rgba[0], rgba[1], rgba[2]);
+        tmp8[1] = draw->full_range ? RGB_TO_U_JPEG(rgba[0], rgba[1], rgba[2]) : RGB_TO_U_CCIR(rgba[0], rgba[1], rgba[2], 0);
+        tmp8[2] = draw->full_range ? RGB_TO_V_JPEG(rgba[0], rgba[1], rgba[2]) : RGB_TO_V_CCIR(rgba[0], rgba[1], rgba[2], 0);
+        tmp8[3] = rgba[3];
     } else if (draw->format == AV_PIX_FMT_GRAY8 || draw->format == AV_PIX_FMT_GRAY8A ||
                draw->format == AV_PIX_FMT_GRAY16LE || draw->format == AV_PIX_FMT_YA16LE ||
                draw->format == AV_PIX_FMT_GRAY9LE  ||
                draw->format == AV_PIX_FMT_GRAY10LE ||
                draw->format == AV_PIX_FMT_GRAY12LE ||
                draw->format == AV_PIX_FMT_GRAY14LE) {
-        const AVPixFmtDescriptor *desc = draw->desc;
-        color->comp[0].u8[0] = RGB_TO_Y_CCIR(rgba[0], rgba[1], rgba[2]);
-        EXPAND(0);
-        color->comp[1].u8[0] = rgba[3];
-        EXPAND(1);
+        tmp8[0] = RGB_TO_Y_CCIR(rgba[0], rgba[1], rgba[2]);
+        tmp8[1] = rgba[3];
     } else {
         av_log(NULL, AV_LOG_WARNING,
                "Color conversion not implemented for %s\n", draw->desc->name);
         memset(color, 128, sizeof(*color));
+        return;
+    }
+
+    for (i = 0; i < desc->nb_components; i++) {
+        if (desc->comp[i].depth > 8)
+            color->comp[desc->comp[i].plane].u16[desc->comp[i].offset / 2] = tmp8[i] <<
+                (draw->desc->comp[i].depth + draw->desc->comp[i].shift - 8);
+        else
+            color->comp[desc->comp[i].plane].u8[desc->comp[i].offset] = tmp8[i];
     }
 }
 
@@ -302,11 +285,6 @@ static void subsampling_bounds(int sub, int *x, int *w, int *start, int *end)
     *w >>= sub;
 }
 
-static int component_used(FFDrawContext *draw, int plane, int comp)
-{
-    return (draw->comp_mask[plane] >> comp) & 1;
-}
-
 /* If alpha is in the [ 0 ; 0x1010101 ] range,
    then alpha * value is in the [ 0 ; 0xFFFFFFFF ] range,
    and >> 24 gives a correct rounding. */
@@ -366,6 +344,9 @@ void ff_blend_rectangle(FFDrawContext *draw, FFDrawColor *color,
     int w_sub, h_sub, x_sub, y_sub, left, right, top, bottom, y;
     uint8_t *p0, *p;
 
+    nb_comp = draw->desc->nb_components -
+        !!(draw->desc->flags & AV_PIX_FMT_FLAG_ALPHA && !(draw->flags & FF_DRAW_PROCESS_ALPHA));
+
     /* TODO optimize if alpha = 0xFF */
     clip_interval(dst_w, &x0, &w, NULL);
     clip_interval(dst_h, &y0, &h, NULL);
@@ -381,7 +362,6 @@ void ff_blend_rectangle(FFDrawContext *draw, FFDrawColor *color,
     nb_planes = draw->nb_planes - !!(draw->desc->flags & AV_PIX_FMT_FLAG_ALPHA && !(draw->flags & FF_DRAW_PROCESS_ALPHA));
     nb_planes += !nb_planes;
     for (plane = 0; plane < nb_planes; plane++) {
-        nb_comp = draw->pixelstep[plane];
         p0 = pointer_at(draw, dst, dst_linesize, plane, x0, y0);
         w_sub = w;
         h_sub = h;
@@ -391,17 +371,19 @@ void ff_blend_rectangle(FFDrawContext *draw, FFDrawColor *color,
         subsampling_bounds(draw->vsub[plane], &y_sub, &h_sub, &top, &bottom);
         for (comp = 0; comp < nb_comp; comp++) {
             const int depth = draw->desc->comp[comp].depth;
+            const int offset = draw->desc->comp[comp].offset;
+            const int index = offset / ((depth + 7) / 8);
 
-            if (!component_used(draw, plane, comp))
+            if (draw->desc->comp[comp].plane != plane)
                 continue;
-            p = p0 + comp;
+            p = p0 + offset;
             if (top) {
                 if (depth <= 8) {
-                    blend_line(p, color->comp[plane].u8[comp], alpha >> 1,
+                    blend_line(p, color->comp[plane].u8[index], alpha >> 1,
                                draw->pixelstep[plane], w_sub,
                                draw->hsub[plane], left, right);
                 } else {
-                    blend_line16(p, color->comp[plane].u16[comp], alpha >> 1,
+                    blend_line16(p, color->comp[plane].u16[index], alpha >> 1,
                                  draw->pixelstep[plane], w_sub,
                                  draw->hsub[plane], left, right);
                 }
@@ -409,14 +391,14 @@ void ff_blend_rectangle(FFDrawContext *draw, FFDrawColor *color,
             }
             if (depth <= 8) {
                 for (y = 0; y < h_sub; y++) {
-                    blend_line(p, color->comp[plane].u8[comp], alpha,
+                    blend_line(p, color->comp[plane].u8[index], alpha,
                                draw->pixelstep[plane], w_sub,
                                draw->hsub[plane], left, right);
                     p += dst_linesize[plane];
                 }
             } else {
                 for (y = 0; y < h_sub; y++) {
-                    blend_line16(p, color->comp[plane].u16[comp], alpha,
+                    blend_line16(p, color->comp[plane].u16[index], alpha,
                                  draw->pixelstep[plane], w_sub,
                                  draw->hsub[plane], left, right);
                     p += dst_linesize[plane];
@@ -424,11 +406,11 @@ void ff_blend_rectangle(FFDrawContext *draw, FFDrawColor *color,
             }
             if (bottom) {
                 if (depth <= 8) {
-                    blend_line(p, color->comp[plane].u8[comp], alpha >> 1,
+                    blend_line(p, color->comp[plane].u8[index], alpha >> 1,
                                draw->pixelstep[plane], w_sub,
                                draw->hsub[plane], left, right);
                 } else {
-                    blend_line16(p, color->comp[plane].u16[comp], alpha >> 1,
+                    blend_line16(p, color->comp[plane].u16[index], alpha >> 1,
                                  draw->pixelstep[plane], w_sub,
                                  draw->hsub[plane], left, right);
                 }
@@ -544,6 +526,9 @@ void ff_blend_mask(FFDrawContext *draw, FFDrawColor *color,
     uint8_t *p0, *p;
     const uint8_t *m;
 
+    nb_comp = draw->desc->nb_components -
+        !!(draw->desc->flags & AV_PIX_FMT_FLAG_ALPHA && !(draw->flags & FF_DRAW_PROCESS_ALPHA));
+
     clip_interval(dst_w, &x0, &mask_w, &xm0);
     clip_interval(dst_h, &y0, &mask_h, &ym0);
     mask += ym0 * mask_linesize;
@@ -559,7 +544,6 @@ void ff_blend_mask(FFDrawContext *draw, FFDrawColor *color,
     nb_planes = draw->nb_planes - !!(draw->desc->flags & AV_PIX_FMT_FLAG_ALPHA && !(draw->flags & FF_DRAW_PROCESS_ALPHA));
     nb_planes += !nb_planes;
     for (plane = 0; plane < nb_planes; plane++) {
-        nb_comp = draw->pixelstep[plane];
         p0 = pointer_at(draw, dst, dst_linesize, plane, x0, y0);
         w_sub = mask_w;
         h_sub = mask_h;
@@ -569,21 +553,23 @@ void ff_blend_mask(FFDrawContext *draw, FFDrawColor *color,
         subsampling_bounds(draw->vsub[plane], &y_sub, &h_sub, &top, &bottom);
         for (comp = 0; comp < nb_comp; comp++) {
             const int depth = draw->desc->comp[comp].depth;
+            const int offset = draw->desc->comp[comp].offset;
+            const int index = offset / ((depth + 7) / 8);
 
-            if (!component_used(draw, plane, comp))
+            if (draw->desc->comp[comp].plane != plane)
                 continue;
-            p = p0 + comp;
+            p = p0 + offset;
             m = mask;
             if (top) {
                 if (depth <= 8) {
                     blend_line_hv(p, draw->pixelstep[plane],
-                                  color->comp[plane].u8[comp], alpha,
+                                  color->comp[plane].u8[index], alpha,
                                   m, mask_linesize, l2depth, w_sub,
                                   draw->hsub[plane], draw->vsub[plane],
                                   xm0, left, right, top);
                 } else {
                     blend_line_hv16(p, draw->pixelstep[plane],
-                                    color->comp[plane].u16[comp], alpha,
+                                    color->comp[plane].u16[index], alpha,
                                     m, mask_linesize, l2depth, w_sub,
                                     draw->hsub[plane], draw->vsub[plane],
                                     xm0, left, right, top);
@@ -594,7 +580,7 @@ void ff_blend_mask(FFDrawContext *draw, FFDrawColor *color,
             if (depth <= 8) {
                 for (y = 0; y < h_sub; y++) {
                     blend_line_hv(p, draw->pixelstep[plane],
-                                  color->comp[plane].u8[comp], alpha,
+                                  color->comp[plane].u8[index], alpha,
                                   m, mask_linesize, l2depth, w_sub,
                                   draw->hsub[plane], draw->vsub[plane],
                                   xm0, left, right, 1 << draw->vsub[plane]);
@@ -604,7 +590,7 @@ void ff_blend_mask(FFDrawContext *draw, FFDrawColor *color,
             } else {
                 for (y = 0; y < h_sub; y++) {
                     blend_line_hv16(p, draw->pixelstep[plane],
-                                    color->comp[plane].u16[comp], alpha,
+                                    color->comp[plane].u16[index], alpha,
                                     m, mask_linesize, l2depth, w_sub,
                                     draw->hsub[plane], draw->vsub[plane],
                                     xm0, left, right, 1 << draw->vsub[plane]);
@@ -615,13 +601,13 @@ void ff_blend_mask(FFDrawContext *draw, FFDrawColor *color,
             if (bottom) {
                 if (depth <= 8) {
                     blend_line_hv(p, draw->pixelstep[plane],
-                                  color->comp[plane].u8[comp], alpha,
+                                  color->comp[plane].u8[index], alpha,
                                   m, mask_linesize, l2depth, w_sub,
                                   draw->hsub[plane], draw->vsub[plane],
                                   xm0, left, right, bottom);
                 } else {
                     blend_line_hv16(p, draw->pixelstep[plane],
-                                    color->comp[plane].u16[comp], alpha,
+                                    color->comp[plane].u16[index], alpha,
                                     m, mask_linesize, l2depth, w_sub,
                                     draw->hsub[plane], draw->vsub[plane],
                                     xm0, left, right, bottom);
diff --git a/libavfilter/drawutils.h b/libavfilter/drawutils.h
index 2ca2475585..396688514e 100644
--- a/libavfilter/drawutils.h
+++ b/libavfilter/drawutils.h
@@ -37,7 +37,6 @@ typedef struct FFDrawContext {
     enum AVPixelFormat format;
     unsigned nb_planes;
     int pixelstep[MAX_PLANES]; /*< offset between pixels */
-    uint8_t comp_mask[MAX_PLANES]; /*< bitmask of used non-alpha components */
     uint8_t hsub[MAX_PLANES];  /*< horizontal subsampling */
     uint8_t vsub[MAX_PLANES];  /*< vertical subsampling */
     uint8_t hsub_max;