imgutils: add function to clear an image to black

Black isn't always just memset(ptr, 0, size). Limited YUV in particular
requires relatively non-obvious values, and filling a frame with
repeating 0 bytes is disallowed in some contexts. With component sizes
larger than 8 or packed YUV, this can become relatively complicated. So
having a generic function for this seems helpful.

In order to handle the complex cases in a generic way without destroying
performance, this code attempts to compute a black pixel, and then uses
that value to clear the image data quickly by using a function like
memset.

Common cases like yuv410p10 or rgba can't be handled with a simple
memset, so there is some code to fill memory with 2/4/8 byte patterns.
For the remaining cases, a generic slow fallback is used.

Signed-off-by: Anton Khirnov <anton@khirnov.net>

4 files changed, 198 insertions, 1 deletions

diff --git a/doc/APIchanges b/doc/APIchanges
index 30a8f809d1..463247f48e 100644
--- a/doc/APIchanges
+++ b/doc/APIchanges
@@ -13,6 +13,9 @@ libavutil:     2017-03-23
 
 API changes, most recent first:
 
+2017-xx-xx - xxxxxxx - lavu 56.4.0 - imgutils.h
+  Add av_image_fill_black().
+
 2017-xx-xx - xxxxxxx - lavu 56.3.0 - frame.h
   Add av_frame_apply_cropping().
 
diff --git a/libavutil/imgutils.c b/libavutil/imgutils.c
index 84abb11656..662962252b 100644
--- a/libavutil/imgutils.c
+++ b/libavutil/imgutils.c
@@ -435,3 +435,170 @@ int av_image_copy_to_buffer(uint8_t *dst, int dst_size,
 
     return size;
 }
+
+// Fill dst[0..dst_size] with the bytes in clear[0..clear_size]. The clear
+// bytes are repeated until dst_size is reached. If dst_size is unaligned (i.e.
+// dst_size%clear_size!=0), the remaining data will be filled with the beginning
+// of the clear data only.
+static void memset_bytes(uint8_t *dst, size_t dst_size, uint8_t *clear,
+                         size_t clear_size)
+{
+    size_t pos = 0;
+    int same = 1;
+    int i;
+
+    if (!clear_size)
+        return;
+
+    // Reduce to memset() if possible.
+    for (i = 0; i < clear_size; i++) {
+        if (clear[i] != clear[0]) {
+            same = 0;
+            break;
+        }
+    }
+    if (same)
+        clear_size = 1;
+
+    if (clear_size == 1) {
+        memset(dst, clear[0], dst_size);
+        dst_size = 0;
+    } else if (clear_size == 2) {
+        uint16_t val = AV_RN16(clear);
+        for (; dst_size >= 2; dst_size -= 2) {
+            AV_WN16(dst, val);
+            dst += 2;
+        }
+    } else if (clear_size == 4) {
+        uint32_t val = AV_RN32(clear);
+        for (; dst_size >= 4; dst_size -= 4) {
+            AV_WN32(dst, val);
+            dst += 4;
+        }
+    } else if (clear_size == 8) {
+        uint32_t val = AV_RN64(clear);
+        for (; dst_size >= 8; dst_size -= 8) {
+            AV_WN64(dst, val);
+            dst += 8;
+        }
+    }
+
+    for (; dst_size; dst_size--)
+        *dst++ = clear[pos++ % clear_size];
+}
+
+// Maximum size in bytes of a plane element (usually a pixel, or multiple pixels
+// if it's a subsampled packed format).
+#define MAX_BLOCK_SIZE 32
+
+int av_image_fill_black(uint8_t *dst_data[4], const ptrdiff_t dst_linesize[4],
+                        enum AVPixelFormat pix_fmt, enum AVColorRange range,
+                        int width, int height)
+{
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
+    int nb_planes = av_pix_fmt_count_planes(pix_fmt);
+    // A pixel or a group of pixels on each plane, with a value that represents black.
+    // Consider e.g. AV_PIX_FMT_UYVY422 for non-trivial cases.
+    uint8_t clear_block[4][MAX_BLOCK_SIZE] = {0}; // clear padding with 0
+    int clear_block_size[4] = {0};
+    ptrdiff_t plane_line_bytes[4] = {0};
+    int rgb, limited;
+    int plane, c;
+
+    if (!desc || nb_planes < 1 || nb_planes > 4 || desc->flags & AV_PIX_FMT_FLAG_HWACCEL)
+        return AVERROR(EINVAL);
+
+    rgb = !!(desc->flags & AV_PIX_FMT_FLAG_RGB);
+    limited = !rgb && range != AVCOL_RANGE_JPEG;
+
+    if (desc->flags & AV_PIX_FMT_FLAG_BITSTREAM) {
+        ptrdiff_t bytewidth = av_image_get_linesize(pix_fmt, width, 0);
+        uint8_t *data;
+        int mono = pix_fmt == AV_PIX_FMT_MONOWHITE || pix_fmt == AV_PIX_FMT_MONOBLACK;
+        int fill = pix_fmt == AV_PIX_FMT_MONOWHITE ? 0xFF : 0;
+        if (nb_planes != 1 || !(rgb || mono) || bytewidth < 1)
+            return AVERROR(EINVAL);
+
+        if (!dst_data)
+            return 0;
+
+        data = dst_data[0];
+
+        // (Bitstream + alpha will be handled incorrectly - it'll remain transparent.)
+        for (;height > 0; height--) {
+            memset(data, fill, bytewidth);
+            data += dst_linesize[0];
+        }
+        return 0;
+    }
+
+    for (c = 0; c < desc->nb_components; c++) {
+        const AVComponentDescriptor comp = desc->comp[c];
+
+        // We try to operate on entire non-subsampled pixel groups (for
+        // AV_PIX_FMT_UYVY422 this would mean two consecutive pixels).
+        clear_block_size[comp.plane] = FFMAX(clear_block_size[comp.plane], comp.step);
+
+        if (clear_block_size[comp.plane] > MAX_BLOCK_SIZE)
+            return AVERROR(EINVAL);
+    }
+
+    // Create a byte array for clearing 1 pixel (sometimes several pixels).
+    for (c = 0; c < desc->nb_components; c++) {
+        const AVComponentDescriptor comp = desc->comp[c];
+        // (Multiple pixels happen e.g. with AV_PIX_FMT_UYVY422.)
+        int w = clear_block_size[comp.plane] / comp.step;
+        uint8_t *c_data[4];
+        const int c_linesize[4] = {0};
+        uint16_t src_array[MAX_BLOCK_SIZE];
+        uint16_t src = 0;
+        int x;
+
+        if (comp.depth > 16)
+            return AVERROR(EINVAL);
+        if (!rgb && comp.depth < 8)
+            return AVERROR(EINVAL);
+        if (w < 1)
+            return AVERROR(EINVAL);
+
+        if (c == 0 && limited) {
+            src = 16 << (comp.depth - 8);
+        } else if ((c == 1 || c == 2) && !rgb) {
+            src = 128 << (comp.depth - 8);
+        } else if (c == 3) {
+            // (Assume even limited YUV uses full range alpha.)
+            src = (1 << comp.depth) - 1;
+        }
+
+        for (x = 0; x < w; x++)
+            src_array[x] = src;
+
+        for (x = 0; x < 4; x++)
+            c_data[x] = &clear_block[x][0];
+
+        av_write_image_line(src_array, c_data, c_linesize, desc, 0, 0, c, w);
+    }
+
+    for (plane = 0; plane < nb_planes; plane++) {
+        plane_line_bytes[plane] = av_image_get_linesize(pix_fmt, width, plane);
+        if (plane_line_bytes[plane] < 0)
+            return AVERROR(EINVAL);
+    }
+
+    if (!dst_data)
+        return 0;
+
+    for (plane = 0; plane < nb_planes; plane++) {
+        size_t bytewidth = plane_line_bytes[plane];
+        uint8_t *data = dst_data[plane];
+        int chroma_div = plane == 1 || plane == 2 ? desc->log2_chroma_h : 0;
+        int plane_h = ((height + ( 1 << chroma_div) - 1)) >> chroma_div;
+
+        for (; plane_h > 0; plane_h--) {
+            memset_bytes(data, bytewidth, &clear_block[plane][0], clear_block_size[plane]);
+            data += dst_linesize[plane];
+        }
+    }
+
+    return 0;
+}
diff --git a/libavutil/imgutils.h b/libavutil/imgutils.h
index fafabc48ca..ae2e6287eb 100644
--- a/libavutil/imgutils.h
+++ b/libavutil/imgutils.h
@@ -225,6 +225,33 @@ int av_image_check_size(unsigned int w, unsigned int h, int log_offset, void *lo
 int av_image_check_sar(unsigned int w, unsigned int h, AVRational sar);
 
 /**
+ * Overwrite the image data with black. This is suitable for filling a
+ * sub-rectangle of an image, meaning the padding between the right most pixel
+ * and the left most pixel on the next line will not be overwritten. For some
+ * formats, the image size might be rounded up due to inherent alignment.
+ *
+ * If the pixel format has alpha, the alpha is cleared to opaque.
+ *
+ * This can return an error if the pixel format is not supported. Normally, all
+ * non-hwaccel pixel formats should be supported.
+ *
+ * Passing NULL for dst_data is allowed. Then the function returns whether the
+ * operation would have succeeded. (It can return an error if the pix_fmt is
+ * not supported.)
+ *
+ * @param dst_data      data pointers to destination image
+ * @param dst_linesize  linesizes for the destination image
+ * @param pix_fmt       the pixel format of the image
+ * @param range         the color range of the image (important for colorspaces such as YUV)
+ * @param width         the width of the image in pixels
+ * @param height        the height of the image in pixels
+ * @return 0 if the image data was cleared, a negative AVERROR code otherwise
+ */
+int av_image_fill_black(uint8_t *dst_data[4], const ptrdiff_t dst_linesize[4],
+                        enum AVPixelFormat pix_fmt, enum AVColorRange range,
+                        int width, int height);
+
+/**
  * @}
  */
 
diff --git a/libavutil/version.h b/libavutil/version.h
index 53bf72efdd..62fb38a27a 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -54,7 +54,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR 56
-#define LIBAVUTIL_VERSION_MINOR  3
+#define LIBAVUTIL_VERSION_MINOR  4
 #define LIBAVUTIL_VERSION_MICRO  0
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \