From de64d8cf171c6ecdca22d57f0bdd7efec95d0c0e Mon Sep 17 00:00:00 2001 From: Aneesh Dogra Date: Fri, 16 Mar 2012 00:43:23 +0530 Subject: qtrle: Use bytestream2 functions to prevent buffer overreads. Signed-off-by: Ronald S. Bultje --- libavcodec/qtrle.c | 187 +++++++++++++++++++++-------------------------------- 1 file changed, 74 insertions(+), 113 deletions(-) (limited to 'libavcodec/qtrle.c') diff --git a/libavcodec/qtrle.c b/libavcodec/qtrle.c index 613dcaea49..df9c8443f2 100644 --- a/libavcodec/qtrle.c +++ b/libavcodec/qtrle.c @@ -35,27 +35,17 @@ #include #include -#include "libavutil/intreadwrite.h" #include "avcodec.h" +#include "bytestream.h" typedef struct QtrleContext { - AVCodecContext *avctx; AVFrame frame; - const unsigned char *buf; - int size; - + GetByteContext g; uint32_t pal[256]; } QtrleContext; -#define CHECK_STREAM_PTR(n) \ - if ((stream_ptr + n) > s->size) { \ - av_log (s->avctx, AV_LOG_INFO, "Problem: stream_ptr out of bounds (%d >= %d)\n", \ - stream_ptr + n, s->size); \ - return; \ - } - #define CHECK_PIXEL_PTR(n) \ if ((pixel_ptr + n > pixel_limit) || (pixel_ptr + n < 0)) { \ av_log (s->avctx, AV_LOG_INFO, "Problem: pixel_ptr = %d, pixel_limit = %d\n", \ @@ -63,7 +53,7 @@ typedef struct QtrleContext { return; \ } \ -static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change) +static void qtrle_decode_1bpp(QtrleContext *s, int row_ptr, int lines_to_change) { int rle_code; int pixel_ptr = 0; @@ -74,9 +64,8 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int int skip; while (lines_to_change) { - CHECK_STREAM_PTR(2); - skip = s->buf[stream_ptr++]; - rle_code = (signed char)s->buf[stream_ptr++]; + skip = bytestream2_get_byte(&s->g); + rle_code = (signed char)bytestream2_get_byte(&s->g); if (rle_code == 0) break; if(skip & 0x80) { @@ -92,9 +81,9 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int rle_code = -rle_code; /* get the next 2 bytes from the stream, treat them as groups * of 8 pixels, and output them rle_code times */ - CHECK_STREAM_PTR(2); - pi0 = s->buf[stream_ptr++]; - pi1 = s->buf[stream_ptr++]; + + pi0 = bytestream2_get_byte(&s->g); + pi1 = bytestream2_get_byte(&s->g); CHECK_PIXEL_PTR(rle_code * 2); while (rle_code--) { @@ -104,17 +93,16 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int } else { /* copy the same pixel directly to output 2 times */ rle_code *= 2; - CHECK_STREAM_PTR(rle_code); CHECK_PIXEL_PTR(rle_code); while (rle_code--) - rgb[pixel_ptr++] = s->buf[stream_ptr++]; + rgb[pixel_ptr++] = bytestream2_get_byte(&s->g); } } } -static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr, - int row_ptr, int lines_to_change, int bpp) +static inline void qtrle_decode_2n4bpp(QtrleContext *s, int row_ptr, + int lines_to_change, int bpp) { int rle_code, i; int pixel_ptr; @@ -125,24 +113,21 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr, int num_pixels = (bpp == 4) ? 8 : 16; while (lines_to_change--) { - CHECK_STREAM_PTR(2); - pixel_ptr = row_ptr + (num_pixels * (s->buf[stream_ptr++] - 1)); + pixel_ptr = row_ptr + (num_pixels * (bytestream2_get_byte(&s->g) - 1)); - while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) { + while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) { if (rle_code == 0) { /* there's another skip code in the stream */ - CHECK_STREAM_PTR(1); - pixel_ptr += (num_pixels * (s->buf[stream_ptr++] - 1)); + pixel_ptr += (num_pixels * (bytestream2_get_byte(&s->g) - 1)); CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */ } else if (rle_code < 0) { /* decode the run length code */ rle_code = -rle_code; /* get the next 4 bytes from the stream, treat them as palette * indexes, and output them rle_code times */ - CHECK_STREAM_PTR(4); for (i = num_pixels-1; i >= 0; i--) { - pi[num_pixels-1-i] = (s->buf[stream_ptr] >> ((i*bpp) & 0x07)) & ((1<>2)-1)) == 0); + pi[num_pixels-1-i] = (bytestream2_peek_byte(&s->g) >> ((i*bpp) & 0x07)) & ((1<g, ((i & ((num_pixels>>2)-1)) == 0)); } CHECK_PIXEL_PTR(rle_code * num_pixels); while (rle_code--) { @@ -152,17 +137,18 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr, } else { /* copy the same pixel directly to output 4 times */ rle_code *= 4; - CHECK_STREAM_PTR(rle_code); CHECK_PIXEL_PTR(rle_code*(num_pixels>>2)); while (rle_code--) { if(bpp == 4) { - rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 4) & 0x0f; - rgb[pixel_ptr++] = (s->buf[stream_ptr++]) & 0x0f; + int x = bytestream2_get_byte(&s->g); + rgb[pixel_ptr++] = (x >> 4) & 0x0f; + rgb[pixel_ptr++] = x & 0x0f; } else { - rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 6) & 0x03; - rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 4) & 0x03; - rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 2) & 0x03; - rgb[pixel_ptr++] = (s->buf[stream_ptr++]) & 0x03; + int x = bytestream2_get_byte(&s->g); + rgb[pixel_ptr++] = (x >> 6) & 0x03; + rgb[pixel_ptr++] = (x >> 4) & 0x03; + rgb[pixel_ptr++] = (x >> 2) & 0x03; + rgb[pixel_ptr++] = x & 0x03; } } } @@ -171,7 +157,7 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr, } } -static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change) +static void qtrle_decode_8bpp(QtrleContext *s, int row_ptr, int lines_to_change) { int rle_code; int pixel_ptr; @@ -181,25 +167,22 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int int pixel_limit = s->frame.linesize[0] * s->avctx->height; while (lines_to_change--) { - CHECK_STREAM_PTR(2); - pixel_ptr = row_ptr + (4 * (s->buf[stream_ptr++] - 1)); + pixel_ptr = row_ptr + (4 * (bytestream2_get_byte(&s->g) - 1)); - while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) { + while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) { if (rle_code == 0) { /* there's another skip code in the stream */ - CHECK_STREAM_PTR(1); - pixel_ptr += (4 * (s->buf[stream_ptr++] - 1)); + pixel_ptr += (4 * (bytestream2_get_byte(&s->g) - 1)); CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */ } else if (rle_code < 0) { /* decode the run length code */ rle_code = -rle_code; /* get the next 4 bytes from the stream, treat them as palette * indexes, and output them rle_code times */ - CHECK_STREAM_PTR(4); - pi1 = s->buf[stream_ptr++]; - pi2 = s->buf[stream_ptr++]; - pi3 = s->buf[stream_ptr++]; - pi4 = s->buf[stream_ptr++]; + pi1 = bytestream2_get_byte(&s->g); + pi2 = bytestream2_get_byte(&s->g); + pi3 = bytestream2_get_byte(&s->g); + pi4 = bytestream2_get_byte(&s->g); CHECK_PIXEL_PTR(rle_code * 4); @@ -212,11 +195,10 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int } else { /* copy the same pixel directly to output 4 times */ rle_code *= 4; - CHECK_STREAM_PTR(rle_code); CHECK_PIXEL_PTR(rle_code); while (rle_code--) { - rgb[pixel_ptr++] = s->buf[stream_ptr++]; + rgb[pixel_ptr++] = bytestream2_get_byte(&s->g); } } } @@ -224,7 +206,7 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int } } -static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change) +static void qtrle_decode_16bpp(QtrleContext *s, int row_ptr, int lines_to_change) { int rle_code; int pixel_ptr; @@ -234,21 +216,17 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int int pixel_limit = s->frame.linesize[0] * s->avctx->height; while (lines_to_change--) { - CHECK_STREAM_PTR(2); - pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 2; + pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 2; - while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) { + while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) { if (rle_code == 0) { /* there's another skip code in the stream */ - CHECK_STREAM_PTR(1); - pixel_ptr += (s->buf[stream_ptr++] - 1) * 2; + pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 2; CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */ } else if (rle_code < 0) { /* decode the run length code */ rle_code = -rle_code; - CHECK_STREAM_PTR(2); - rgb16 = AV_RB16(&s->buf[stream_ptr]); - stream_ptr += 2; + rgb16 = bytestream2_get_be16(&s->g); CHECK_PIXEL_PTR(rle_code * 2); @@ -257,13 +235,11 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int pixel_ptr += 2; } } else { - CHECK_STREAM_PTR(rle_code * 2); CHECK_PIXEL_PTR(rle_code * 2); /* copy pixels directly to output */ while (rle_code--) { - rgb16 = AV_RB16(&s->buf[stream_ptr]); - stream_ptr += 2; + rgb16 = bytestream2_get_be16(&s->g); *(unsigned short *)(&rgb[pixel_ptr]) = rgb16; pixel_ptr += 2; } @@ -273,7 +249,7 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int } } -static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change) +static void qtrle_decode_24bpp(QtrleContext *s, int row_ptr, int lines_to_change) { int rle_code; int pixel_ptr; @@ -283,22 +259,19 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int int pixel_limit = s->frame.linesize[0] * s->avctx->height; while (lines_to_change--) { - CHECK_STREAM_PTR(2); - pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 3; + pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 3; - while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) { + while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) { if (rle_code == 0) { /* there's another skip code in the stream */ - CHECK_STREAM_PTR(1); - pixel_ptr += (s->buf[stream_ptr++] - 1) * 3; + pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 3; CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */ } else if (rle_code < 0) { /* decode the run length code */ rle_code = -rle_code; - CHECK_STREAM_PTR(3); - r = s->buf[stream_ptr++]; - g = s->buf[stream_ptr++]; - b = s->buf[stream_ptr++]; + r = bytestream2_get_byte(&s->g); + g = bytestream2_get_byte(&s->g); + b = bytestream2_get_byte(&s->g); CHECK_PIXEL_PTR(rle_code * 3); @@ -308,14 +281,13 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int rgb[pixel_ptr++] = b; } } else { - CHECK_STREAM_PTR(rle_code * 3); CHECK_PIXEL_PTR(rle_code * 3); /* copy pixels directly to output */ while (rle_code--) { - rgb[pixel_ptr++] = s->buf[stream_ptr++]; - rgb[pixel_ptr++] = s->buf[stream_ptr++]; - rgb[pixel_ptr++] = s->buf[stream_ptr++]; + rgb[pixel_ptr++] = bytestream2_get_byte(&s->g); + rgb[pixel_ptr++] = bytestream2_get_byte(&s->g); + rgb[pixel_ptr++] = bytestream2_get_byte(&s->g); } } } @@ -323,7 +295,7 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int } } -static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change) +static void qtrle_decode_32bpp(QtrleContext *s, int row_ptr, int lines_to_change) { int rle_code; int pixel_ptr; @@ -333,21 +305,17 @@ static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int int pixel_limit = s->frame.linesize[0] * s->avctx->height; while (lines_to_change--) { - CHECK_STREAM_PTR(2); - pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 4; + pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 4; - while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) { + while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) { if (rle_code == 0) { /* there's another skip code in the stream */ - CHECK_STREAM_PTR(1); - pixel_ptr += (s->buf[stream_ptr++] - 1) * 4; + pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 4; CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */ } else if (rle_code < 0) { /* decode the run length code */ rle_code = -rle_code; - CHECK_STREAM_PTR(4); - argb = AV_RB32(s->buf + stream_ptr); - stream_ptr += 4; + argb = bytestream2_get_be32(&s->g); CHECK_PIXEL_PTR(rle_code * 4); @@ -356,14 +324,12 @@ static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int pixel_ptr += 4; } } else { - CHECK_STREAM_PTR(rle_code * 4); CHECK_PIXEL_PTR(rle_code * 4); /* copy pixels directly to output */ while (rle_code--) { - argb = AV_RB32(s->buf + stream_ptr); + argb = bytestream2_get_be32(&s->g); AV_WN32A(rgb + pixel_ptr, argb); - stream_ptr += 4; pixel_ptr += 4; } } @@ -419,16 +385,12 @@ static int qtrle_decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPacket *avpkt) { - const uint8_t *buf = avpkt->data; - int buf_size = avpkt->size; QtrleContext *s = avctx->priv_data; int header, start_line; - int stream_ptr, height, row_ptr; + int height, row_ptr; int has_palette = 0; - s->buf = buf; - s->size = buf_size; - + bytestream2_init(&s->g, avpkt->data, avpkt->size); s->frame.reference = 1; s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE | FF_BUFFER_HINTS_READABLE; @@ -438,64 +400,63 @@ static int qtrle_decode_frame(AVCodecContext *avctx, } /* check if this frame is even supposed to change */ - if (s->size < 8) + if (avpkt->size < 8) goto done; /* start after the chunk size */ - stream_ptr = 4; + bytestream2_seek(&s->g, 4, SEEK_SET); /* fetch the header */ - header = AV_RB16(&s->buf[stream_ptr]); - stream_ptr += 2; + header = bytestream2_get_be16(&s->g); /* if a header is present, fetch additional decoding parameters */ if (header & 0x0008) { - if(s->size < 14) + if (avpkt->size < 14) goto done; - start_line = AV_RB16(&s->buf[stream_ptr]); - stream_ptr += 4; - height = AV_RB16(&s->buf[stream_ptr]); - stream_ptr += 4; + start_line = bytestream2_get_be16(&s->g); + bytestream2_skip(&s->g, 2); + height = bytestream2_get_be16(&s->g); + bytestream2_skip(&s->g, 2); } else { start_line = 0; - height = s->avctx->height; + height = s->avctx->height; } row_ptr = s->frame.linesize[0] * start_line; switch (avctx->bits_per_coded_sample) { case 1: case 33: - qtrle_decode_1bpp(s, stream_ptr, row_ptr, height); + qtrle_decode_1bpp(s, row_ptr, height); break; case 2: case 34: - qtrle_decode_2n4bpp(s, stream_ptr, row_ptr, height, 2); + qtrle_decode_2n4bpp(s, row_ptr, height, 2); has_palette = 1; break; case 4: case 36: - qtrle_decode_2n4bpp(s, stream_ptr, row_ptr, height, 4); + qtrle_decode_2n4bpp(s, row_ptr, height, 4); has_palette = 1; break; case 8: case 40: - qtrle_decode_8bpp(s, stream_ptr, row_ptr, height); + qtrle_decode_8bpp(s, row_ptr, height); has_palette = 1; break; case 16: - qtrle_decode_16bpp(s, stream_ptr, row_ptr, height); + qtrle_decode_16bpp(s, row_ptr, height); break; case 24: - qtrle_decode_24bpp(s, stream_ptr, row_ptr, height); + qtrle_decode_24bpp(s, row_ptr, height); break; case 32: - qtrle_decode_32bpp(s, stream_ptr, row_ptr, height); + qtrle_decode_32bpp(s, row_ptr, height); break; default: @@ -521,7 +482,7 @@ done: *(AVFrame*)data = s->frame; /* always report that the buffer was completely consumed */ - return buf_size; + return avpkt->size; } static av_cold int qtrle_decode_end(AVCodecContext *avctx) -- cgit v1.2.3