Ensure that the palette is set in data[1] for all 8bit formats.

Also document it.

Originally committed as revision 17018 to svn://svn.ffmpeg.org/ffmpeg/trunk

3 files changed, 58 insertions, 14 deletions

diff --git a/libavcodec/imgconvert.c b/libavcodec/imgconvert.c
index a78b454a67..7e52ea384d 100644
--- a/libavcodec/imgconvert.c
+++ b/libavcodec/imgconvert.c
@@ -443,6 +443,43 @@ void avcodec_pix_fmt_string (char *buf, int buf_size, int pix_fmt)
     }
 }
 
+int ff_set_systematic_pal(uint32_t pal[256], enum PixelFormat pix_fmt){
+    int i;
+
+    for(i=0; i<256; i++){
+        int r,g,b;
+
+        switch(pix_fmt) {
+        case PIX_FMT_RGB8:
+            r= (i>>5    )*36;
+            g= ((i>>2)&7)*36;
+            b= (i&3     )*85;
+            break;
+        case PIX_FMT_BGR8:
+            b= (i>>6    )*85;
+            g= ((i>>3)&7)*36;
+            r= (i&7     )*36;
+            break;
+        case PIX_FMT_RGB4_BYTE:
+            r= (i>>3    )*255;
+            g= ((i>>1)&3)*85;
+            b= (i&1     )*255;
+            break;
+        case PIX_FMT_BGR4_BYTE:
+            b= (i>>3    )*255;
+            g= ((i>>1)&3)*85;
+            r= (i&1     )*255;
+            break;
+        case PIX_FMT_GRAY8:
+            r=b=g= i;
+            break;
+        }
+        pal[i] =  b + (g<<8) + (r<<16);
+    }
+
+    return 0;
+}
+
 int ff_fill_linesize(AVPicture *picture, int pix_fmt, int width)
 {
     int w2;
@@ -505,13 +542,6 @@ int ff_fill_linesize(AVPicture *picture, int pix_fmt, int width)
     case PIX_FMT_UYYVYY411:
         picture->linesize[0] = width + width/2;
         break;
-    case PIX_FMT_RGB8:
-    case PIX_FMT_BGR8:
-    case PIX_FMT_RGB4_BYTE:
-    case PIX_FMT_BGR4_BYTE:
-    case PIX_FMT_GRAY8:
-        picture->linesize[0] = width;
-        break;
     case PIX_FMT_RGB4:
     case PIX_FMT_BGR4:
         picture->linesize[0] = width / 2;
@@ -521,6 +551,11 @@ int ff_fill_linesize(AVPicture *picture, int pix_fmt, int width)
         picture->linesize[0] = (width + 7) >> 3;
         break;
     case PIX_FMT_PAL8:
+    case PIX_FMT_RGB8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB4_BYTE:
+    case PIX_FMT_BGR4_BYTE:
+    case PIX_FMT_GRAY8:
         picture->linesize[0] = width;
         picture->linesize[1] = 4;
         break;
@@ -588,11 +623,6 @@ int ff_fill_pointer(AVPicture *picture, uint8_t *ptr, int pix_fmt,
     case PIX_FMT_YUYV422:
     case PIX_FMT_UYVY422:
     case PIX_FMT_UYYVYY411:
-    case PIX_FMT_RGB8:
-    case PIX_FMT_BGR8:
-    case PIX_FMT_RGB4_BYTE:
-    case PIX_FMT_BGR4_BYTE:
-    case PIX_FMT_GRAY8:
     case PIX_FMT_RGB4:
     case PIX_FMT_BGR4:
     case PIX_FMT_MONOWHITE:
@@ -603,6 +633,11 @@ int ff_fill_pointer(AVPicture *picture, uint8_t *ptr, int pix_fmt,
         picture->data[3] = NULL;
         return size;
     case PIX_FMT_PAL8:
+    case PIX_FMT_RGB8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB4_BYTE:
+    case PIX_FMT_BGR4_BYTE:
+    case PIX_FMT_GRAY8:
         size2 = (size + 3) & ~3;
         picture->data[0] = ptr;
         picture->data[1] = ptr + size2; /* palette is stored here as 256 32 bit words */
@@ -2000,6 +2035,9 @@ int avpicture_alloc(AVPicture *picture,
     if (!ptr)
         goto fail;
     avpicture_fill(picture, ptr, pix_fmt, width, height);
+    if(picture->data[1] && !picture->data[2])
+        ff_set_systematic_pal((uint32_t*)picture->data[1], pix_fmt);
+
     return 0;
  fail:
     memset(picture, 0, sizeof(AVPicture));
diff --git a/libavcodec/imgconvert.h b/libavcodec/imgconvert.h
index 83bce68f56..3b5bed76af 100644
--- a/libavcodec/imgconvert.h
+++ b/libavcodec/imgconvert.h
@@ -33,6 +33,8 @@ int ff_fill_pointer(AVPicture *picture, uint8_t *ptr, int pix_fmt, int height);
 
 int ff_get_plane_bytewidth(enum PixelFormat pix_fmt, int width, int plane);
 
+int ff_set_systematic_pal(uint32_t pal[256], enum PixelFormat pix_fmt);
+
 int img_convert(AVPicture *dst, int dst_pix_fmt, const AVPicture *src,
                 int src_pix_fmt, int src_width, int src_height);
 
diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 292698c336..26ce057e3c 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -154,6 +154,8 @@ void avcodec_align_dimensions(AVCodecContext *s, int *width, int *height){
             h_align=4;
         }
     case PIX_FMT_PAL8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB8:
         if(s->codec_id == CODEC_ID_SMC){
             w_align=4;
             h_align=4;
@@ -281,12 +283,14 @@ int avcodec_default_get_buffer(AVCodecContext *s, AVFrame *pic){
             if(buf->base[i]==NULL) return -1;
             memset(buf->base[i], 128, size[i]);
 
-            // no edge if EDEG EMU or not planar YUV, we check for PAL8 redundantly to protect against a exploitable bug regression ...
-            if((s->flags&CODEC_FLAG_EMU_EDGE) || (s->pix_fmt == PIX_FMT_PAL8) || !size[2])
+            // no edge if EDEG EMU or not planar YUV
+            if((s->flags&CODEC_FLAG_EMU_EDGE) || !size[2])
                 buf->data[i] = buf->base[i];
             else
                 buf->data[i] = buf->base[i] + ALIGN((buf->linesize[i]*EDGE_WIDTH>>v_shift) + (EDGE_WIDTH>>h_shift), stride_align[i]);
         }
+        if(size[1] && !size[2])
+            ff_set_systematic_pal((uint32_t*)buf->data[1], s->pix_fmt);
         buf->width  = s->width;
         buf->height = s->height;
         buf->pix_fmt= s->pix_fmt;