iff: support masking plane feature

Fixes ticket 705.

Reported-by: ami-stuff
Signed-off-by: Peter Ross <pross@xvid.org>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 54 insertions, 4 deletions

diff --git a/libavcodec/iff.c b/libavcodec/iff.c
index c0e4eb4d38..24167c722c 100644
--- a/libavcodec/iff.c
+++ b/libavcodec/iff.c
@@ -44,6 +44,8 @@ typedef struct {
     uint8_t * planebuf;
     uint8_t * ham_buf;      ///< temporary buffer for planar to chunky conversation
     uint32_t *ham_palbuf;   ///< HAM decode table
+    uint32_t *mask_buf;     ///< temporary buffer for palette indices
+    uint32_t *mask_palbuf;  ///< masking palette table
     unsigned  compression;  ///< delta compression method used
     unsigned  bpp;          ///< bits per plane to decode (differs from bits_per_coded_sample if HAM)
     unsigned  ham;          ///< 0 if non-HAM or number of hold bits (6 for bpp > 6, 4 otherwise)
@@ -157,6 +159,7 @@ static int ff_cmap_read_palette(AVCodecContext *avctx, uint32_t *pal)
         if (s->flags && count >= 32) { // EHB
             for (i = 0; i < 32; i++)
                 pal[i + 32] = 0xFF000000 | (AV_RB24(palette + i*3) & 0xFEFEFE) >> 1;
+            count = FFMAX(count, 64);
         }
     } else { // Create gray-scale color palette for bps < 8
         count = 1 << avctx->bits_per_coded_sample;
@@ -165,7 +168,11 @@ static int ff_cmap_read_palette(AVCodecContext *avctx, uint32_t *pal)
             pal[i] = 0xFF000000 | gray2rgb((i * 255) >> avctx->bits_per_coded_sample);
         }
     }
-    if (s->masking == MASK_HAS_TRANSPARENT_COLOR &&
+    if (s->masking == MASK_HAS_MASK) {
+        memcpy(pal + (1 << avctx->bits_per_coded_sample), pal, count * 4);
+        for (i = 0; i < count; i++)
+            pal[i] &= 0xFFFFFF;
+    } else if (s->masking == MASK_HAS_TRANSPARENT_COLOR &&
         s->transparency < 1 << avctx->bits_per_coded_sample)
         pal[s->transparency] &= 0xFFFFFF;
     return 0;
@@ -219,7 +226,22 @@ static int extract_header(AVCodecContext *const avctx,
         s->flags        = bytestream_get_byte(&buf);
         s->transparency = bytestream_get_be16(&buf);
         s->masking      = bytestream_get_byte(&buf);
-        if (s->masking != MASK_NONE && s->masking != MASK_HAS_TRANSPARENT_COLOR) {
+        if (s->masking == MASK_HAS_MASK) {
+            if (s->bpp >= 8) {
+                avctx->pix_fmt = PIX_FMT_RGB32;
+                av_freep(&s->mask_buf);
+                av_freep(&s->mask_palbuf);
+                s->mask_buf = av_malloc((s->planesize * 32) + FF_INPUT_BUFFER_PADDING_SIZE);
+                if (!s->mask_buf)
+                    return AVERROR(ENOMEM);
+                s->mask_palbuf = av_malloc((2 << s->bpp) * sizeof(uint32_t) + FF_INPUT_BUFFER_PADDING_SIZE);
+                if (!s->mask_palbuf) {
+                    av_freep(&s->mask_buf);
+                    return AVERROR(ENOMEM);
+                }
+            }
+            s->bpp++;
+        } else if (s->masking != MASK_NONE && s->masking != MASK_HAS_TRANSPARENT_COLOR) {
             av_log(avctx, AV_LOG_ERROR, "Masking not supported\n");
             return AVERROR_PATCHWELCOME;
         }
@@ -236,12 +258,15 @@ static int extract_header(AVCodecContext *const avctx,
 
         if (s->ham) {
             int i, count = FFMIN(palette_size / 3, 1 << s->ham);
+            int ham_count;
             const uint8_t *const palette = avctx->extradata + AV_RB16(avctx->extradata);
+
             s->ham_buf = av_malloc((s->planesize * 8) + FF_INPUT_BUFFER_PADDING_SIZE);
             if (!s->ham_buf)
                 return AVERROR(ENOMEM);
 
-            s->ham_palbuf = av_malloc((8 * (1 << s->ham) * sizeof (uint32_t)) + FF_INPUT_BUFFER_PADDING_SIZE);
+            ham_count = 8 * (1 << s->ham);
+            s->ham_palbuf = av_malloc((ham_count << !!(s->masking == MASK_HAS_MASK)) * sizeof (uint32_t) + FF_INPUT_BUFFER_PADDING_SIZE);
             if (!s->ham_palbuf) {
                 av_freep(&s->ham_buf);
                 return AVERROR(ENOMEM);
@@ -271,6 +296,10 @@ static int extract_header(AVCodecContext *const avctx,
                 s->ham_palbuf[(i+count*2)*2+1] = tmp;
                 s->ham_palbuf[(i+count*3)*2+1] = tmp << 8;
             }
+            if (s->masking == MASK_HAS_MASK) {
+                for (i = 0; i < ham_count; i++)
+                    s->ham_palbuf[(1 << s->bpp) + i] = s->ham_palbuf[i] | 0xFF000000;
+            }
         }
     }
 
@@ -384,6 +413,14 @@ static void decode_ham_plane32(uint32_t *dst, const uint8_t  *buf,
     } while (--buf_size);
 }
 
+static void lookup_pal_indicies(uint32_t *dst, const uint32_t *buf,
+                         const uint32_t *const pal, unsigned width)
+{
+    do {
+        *dst++ = pal[*buf++];
+    } while (--width);
+}
+
 /**
  * Decode one complete byterun1 encoded line.
  *
@@ -536,9 +573,12 @@ static int decode_frame_byterun1(AVCodecContext *avctx,
     } else if ((res = avctx->get_buffer(avctx, &s->frame)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
         return res;
-    } else if (avctx->bits_per_coded_sample <= 8 && avctx->pix_fmt != PIX_FMT_GRAY8) {
+    } else if (avctx->pix_fmt == PIX_FMT_PAL8) {
         if ((res = ff_cmap_read_palette(avctx, (uint32_t*)s->frame.data[1])) < 0)
             return res;
+    } else if (avctx->pix_fmt == PIX_FMT_RGB32 && avctx->bits_per_coded_sample <= 8) {
+        if ((res = ff_cmap_read_palette(avctx, s->mask_palbuf)) < 0)
+            return res;
     }
     s->init = 1;
 
@@ -552,6 +592,16 @@ static int decode_frame_byterun1(AVCodecContext *avctx,
                     decodeplane8(row, s->planebuf, s->planesize, plane);
                 }
             }
+        } else if (avctx->bits_per_coded_sample <= 8) { //8-bit (+ mask) to PIX_FMT_BGR32
+            for (y = 0; y < avctx->height ; y++ ) {
+                uint8_t *row = &s->frame.data[0][y*s->frame.linesize[0]];
+                memset(s->mask_buf, 0, avctx->width * sizeof(uint32_t));
+                for (plane = 0; plane < s->bpp; plane++) {
+                    buf += decode_byterun(s->planebuf, s->planesize, buf, buf_end);
+                    decodeplane32(s->mask_buf, s->planebuf, s->planesize, plane);
+                }
+                lookup_pal_indicies((uint32_t *) row, s->mask_buf, s->mask_palbuf, avctx->width);
+            }
         } else if (s->ham) { // HAM to PIX_FMT_BGR32
             for (y = 0; y < avctx->height ; y++) {
                 uint8_t *row = &s->frame.data[0][y*s->frame.linesize[0]];