From d613091f8858d87789916e2bd7a84ea3144077d4 Mon Sep 17 00:00:00 2001
From: Vittorio Giovara <vittorio.giovara@gmail.com>
Date: Thu, 13 Mar 2014 15:14:37 +0100
Subject: sgi: decode 16bit RLE images

---
 libavcodec/sgidec.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 6 deletions(-)

(limited to 'libavcodec/sgidec.c')

diff --git a/libavcodec/sgidec.c b/libavcodec/sgidec.c
index 357d54e743..ebca2e8f00 100644
--- a/libavcodec/sgidec.c
+++ b/libavcodec/sgidec.c
@@ -43,8 +43,8 @@ typedef struct SgiState {
  * @param pixelstride pixel stride of input buffer
  * @return size of output in bytes, -1 if buffer overflows
  */
-static int expand_rle_row(SgiState *s, uint8_t *out_buf,
-                          int len, int pixelstride)
+static int expand_rle_row8(SgiState *s, uint8_t *out_buf,
+                           int len, int pixelstride)
 {
     unsigned char pixel, count;
     unsigned char *orig = out_buf;
@@ -81,6 +81,46 @@ static int expand_rle_row(SgiState *s, uint8_t *out_buf,
     return (out_buf - orig) / pixelstride;
 }
 
+static int expand_rle_row16(SgiState *s, uint16_t *out_buf,
+                            int len, int pixelstride)
+{
+    unsigned short pixel;
+    unsigned char count;
+    unsigned short *orig = out_buf;
+    uint16_t *out_end = out_buf + len;
+
+    while (out_buf < out_end) {
+        if (bytestream2_get_bytes_left(&s->g) < 2)
+            return AVERROR_INVALIDDATA;
+        pixel = bytestream2_get_be16u(&s->g);
+        if (!(count = (pixel & 0x7f)))
+            break;
+
+        /* Check for buffer overflow. */
+        if (pixelstride * (count - 1) >= len) {
+            av_log(s->avctx, AV_LOG_ERROR, "Invalid pixel count.\n");
+            return AVERROR_INVALIDDATA;
+        }
+
+        if (pixel & 0x80) {
+            while (count--) {
+                pixel = bytestream2_get_ne16(&s->g);
+                AV_WN16A(out_buf, pixel);
+                out_buf += pixelstride;
+            }
+        } else {
+            pixel = bytestream2_get_ne16(&s->g);
+
+            while (count--) {
+                AV_WN16A(out_buf, pixel);
+                out_buf += pixelstride;
+            }
+        }
+    }
+    return (out_buf - orig) / pixelstride;
+}
+
+
 /**
  * Read a run length encoded SGI image.
  * @param out_buf output buffer
@@ -94,6 +134,7 @@ static int read_rle_sgi(uint8_t *out_buf, SgiState *s)
     GetByteContext g_table = s->g;
     unsigned int y, z;
     unsigned int start_offset;
+    int linesize, ret;
 
     /* size of  RLE offset and length tables */
     if (len * 2 > bytestream2_get_bytes_left(&s->g)) {
@@ -103,13 +144,16 @@ static int read_rle_sgi(uint8_t *out_buf, SgiState *s)
     for (z = 0; z < s->depth; z++) {
         dest_row = out_buf;
         for (y = 0; y < s->height; y++) {
+            linesize = s->width * s->depth * s->bytes_per_channel;
             dest_row -= s->linesize;
             start_offset = bytestream2_get_be32(&g_table);
             bytestream2_seek(&s->g, start_offset, SEEK_SET);
-            if (expand_rle_row(s, dest_row + z, s->width * s->depth,
-                               s->depth) != s->width) {
+            if (s->bytes_per_channel == 1)
+                ret = expand_rle_row8(s, dest_row + z, linesize, s->depth);
+            else
+                ret = expand_rle_row16(s, (uint16_t *)dest_row + z, linesize, s->depth);
+            if (ret != s->width)
                 return AVERROR_INVALIDDATA;
-            }
         }
     }
     return 0;
@@ -183,7 +227,7 @@ static int decode_frame(AVCodecContext *avctx,
     s->height            = bytestream2_get_be16(&s->g);
     s->depth             = bytestream2_get_be16(&s->g);
 
-    if (s->bytes_per_channel != 1 && (s->bytes_per_channel != 2 || rle)) {
+    if (s->bytes_per_channel != 1 && s->bytes_per_channel != 2) {
         av_log(avctx, AV_LOG_ERROR, "wrong channel number\n");
         return -1;
     }
-- 
cgit v1.2.3