From 9bc055e358c6bd9fb517a0f0ecf299b83f2d8655 Mon Sep 17 00:00:00 2001
From: Alex Converse <alex.converse@gmail.com>
Date: Wed, 19 Sep 2012 11:10:26 -0700
Subject: tiffenc: Simplify pixel format setup using AVPixFmtDescriptor.

---
 libavcodec/tiffenc.c | 40 +++++++++++++++++-----------------------
 1 file changed, 17 insertions(+), 23 deletions(-)

(limited to 'libavcodec')

diff --git a/libavcodec/tiffenc.c b/libavcodec/tiffenc.c
index 96a6f0b1f3..4e64201a89 100644
--- a/libavcodec/tiffenc.c
+++ b/libavcodec/tiffenc.c
@@ -27,6 +27,7 @@
 
 #include "libavutil/log.h"
 #include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
 
 #include "avcodec.h"
 #include "config.h"
@@ -219,6 +220,7 @@ static int encode_frame(AVCodecContext * avctx, AVPacket *pkt,
     int is_yuv = 0;
     uint8_t *yuv_line = NULL;
     int shift_h, shift_v;
+    const AVPixFmtDescriptor* pfd;
 
     s->avctx = avctx;
 
@@ -232,41 +234,35 @@ static int encode_frame(AVCodecContext * avctx, AVPacket *pkt,
     s->subsampling[0] = 1;
     s->subsampling[1] = 1;
 
-    s->bpp_tab_size = 0;
     switch (avctx->pix_fmt) {
     case PIX_FMT_RGB48LE:
-        s->bpp = 48;
-        s->photometric_interpretation = 2;
-        s->bpp_tab_size = 3;
-        for (i = 0; i < s->bpp_tab_size; i++) {
-            bpp_tab[i] = 16;
-        }
-        break;
     case PIX_FMT_GRAY16LE:
-        s->bpp = 16;
-        s->photometric_interpretation = 1;
-        s->bpp_tab_size = 1;
-        bpp_tab[0] = 16;
-        break;
     case PIX_FMT_RGB24:
-        s->bpp = 24;
-        s->photometric_interpretation = 2;
-        break;
     case PIX_FMT_GRAY8:
-        s->bpp = 8;
-        s->photometric_interpretation = 1;
-        break;
     case PIX_FMT_PAL8:
-        s->bpp = 8;
-        s->photometric_interpretation = 3;
+        pfd = &av_pix_fmt_descriptors[avctx->pix_fmt];
+        s->bpp = av_get_bits_per_pixel(pfd);
+        if (pfd->flags & PIX_FMT_PAL) {
+            s->photometric_interpretation = 3;
+        } else if (pfd->flags & PIX_FMT_RGB) {
+            s->photometric_interpretation = 2;
+        } else {
+            s->photometric_interpretation = 1;
+        }
+        s->bpp_tab_size = pfd->nb_components;
+        for (i = 0; i < s->bpp_tab_size; i++) {
+            bpp_tab[i] = s->bpp / s->bpp_tab_size;
+        }
         break;
     case PIX_FMT_MONOBLACK:
         s->bpp = 1;
         s->photometric_interpretation = 1;
+        s->bpp_tab_size = 0;
         break;
     case PIX_FMT_MONOWHITE:
         s->bpp = 1;
         s->photometric_interpretation = 0;
+        s->bpp_tab_size = 0;
         break;
     case PIX_FMT_YUV420P:
     case PIX_FMT_YUV422P:
@@ -287,8 +283,6 @@ static int encode_frame(AVCodecContext * avctx, AVPacket *pkt,
                "This colors format is not supported\n");
         return -1;
     }
-    if (!s->bpp_tab_size)
-        s->bpp_tab_size = (s->bpp >> 3);
 
     if (s->compr == TIFF_DEFLATE || s->compr == TIFF_ADOBE_DEFLATE || s->compr == TIFF_LZW)
         //best choose for DEFLATE
-- 
cgit v1.2.3


From b92dfb56d4582633571db18c3d904f8602eaa2a6 Mon Sep 17 00:00:00 2001
From: Alex Converse <alex.converse@gmail.com>
Date: Wed, 19 Sep 2012 11:12:58 -0700
Subject: tiffenc: Check av_malloc() results.

---
 libavcodec/tiffenc.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

(limited to 'libavcodec')

diff --git a/libavcodec/tiffenc.c b/libavcodec/tiffenc.c
index 4e64201a89..e6fd6cf69a 100644
--- a/libavcodec/tiffenc.c
+++ b/libavcodec/tiffenc.c
@@ -316,6 +316,10 @@ static int encode_frame(AVCodecContext * avctx, AVPacket *pkt,
 
     strip_sizes = av_mallocz(sizeof(*strip_sizes) * strips);
     strip_offsets = av_mallocz(sizeof(*strip_offsets) * strips);
+    if (!strip_sizes || !strip_offsets) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
 
     bytes_per_row = (((s->width - 1)/s->subsampling[0] + 1) * s->bpp
                     * s->subsampling[0] * s->subsampling[1] + 7) >> 3;
@@ -323,6 +327,7 @@ static int encode_frame(AVCodecContext * avctx, AVPacket *pkt,
         yuv_line = av_malloc(bytes_per_row);
         if (yuv_line == NULL){
             av_log(s->avctx, AV_LOG_ERROR, "Not enough memory\n");
+            ret = AVERROR(ENOMEM);
             goto fail;
         }
     }
@@ -335,6 +340,10 @@ static int encode_frame(AVCodecContext * avctx, AVPacket *pkt,
 
         zlen = bytes_per_row * s->rps;
         zbuf = av_malloc(zlen);
+        if (!zbuf) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
         strip_offsets[0] = ptr - pkt->data;
         zn = 0;
         for (j = 0; j < s->rps; j++) {
@@ -359,8 +368,13 @@ static int encode_frame(AVCodecContext * avctx, AVPacket *pkt,
     } else
 #endif
     {
-        if(s->compr == TIFF_LZW)
+        if (s->compr == TIFF_LZW) {
             s->lzws = av_malloc(ff_lzw_encode_state_size);
+            if (!s->lzws) {
+                ret = AVERROR(ENOMEM);
+                goto fail;
+            }
+        }
         for (i = 0; i < s->height; i++) {
             if (strip_sizes[i / s->rps] == 0) {
                 if(s->compr == TIFF_LZW){
-- 
cgit v1.2.3