diff options
| author | Ronald S. Bultje <rsbultje@gmail.com> | 2011-10-03 08:38:03 -0700 |
|---|---|---|
| committer | Ronald S. Bultje <rsbultje@gmail.com> | 2011-10-05 04:01:23 -0700 |
| commit | 330deb75923675224fb9aed311d3d6ce3ec52420 (patch) | |
| tree | e1ab909e74b0c360de911c0e770f116fb89b5a0b | |
| parent | 0884dd5a1b87aff6c8a06e6492dece5cef8f3978 (diff) | |
mpegvideo: set correct offset for edge emulation buffer.
Using the old code, half of it was unused and the other half was too
small for e.g. >8bpp interlaced data, causing random buffer overruns.
| -rw-r--r-- | libavcodec/mpegvideo.c | 6 | ||||
| -rw-r--r-- | libavcodec/mpegvideo.h | 3 |
2 files changed, 3 insertions, 6 deletions
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index baab3c85e6..b8ac2ceb3d 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -377,8 +377,7 @@ static int init_duplicate_context(MpegEncContext *s, MpegEncContext *base){ int i; // edge emu needs blocksize + filter length - 1 (=17x17 for halfpel / 21x21 for h264) - FF_ALLOCZ_OR_GOTO(s->avctx, s->allocated_edge_emu_buffer, (s->width+64)*2*21*2, fail); //(width + edge + align)*interlaced*MBsize*tolerance - s->edge_emu_buffer= s->allocated_edge_emu_buffer + (s->width+64)*2*21; + FF_ALLOCZ_OR_GOTO(s->avctx, s->edge_emu_buffer, (s->width+64)*2*21*2, fail); //(width + edge + align)*interlaced*MBsize*tolerance //FIXME should be linesize instead of s->width*2 but that is not known before get_buffer() FF_ALLOCZ_OR_GOTO(s->avctx, s->me.scratchpad, (s->width+64)*4*16*2*sizeof(uint8_t), fail) @@ -416,7 +415,7 @@ fail: static void free_duplicate_context(MpegEncContext *s){ if(s==NULL) return; - av_freep(&s->allocated_edge_emu_buffer); s->edge_emu_buffer= NULL; + av_freep(&s->edge_emu_buffer); av_freep(&s->me.scratchpad); s->me.temp= s->rd_scratchpad= @@ -433,7 +432,6 @@ static void free_duplicate_context(MpegEncContext *s){ static void backup_duplicate_context(MpegEncContext *bak, MpegEncContext *src){ #define COPY(a) bak->a= src->a - COPY(allocated_edge_emu_buffer); COPY(edge_emu_buffer); COPY(me.scratchpad); COPY(me.temp); diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h index 923d46c884..d0f4bfdc6c 100644 --- a/libavcodec/mpegvideo.h +++ b/libavcodec/mpegvideo.h @@ -319,8 +319,7 @@ typedef struct MpegEncContext { uint8_t *mbintra_table; ///< used to avoid setting {ac, dc, cbp}-pred stuff to zero on inter MB decoding uint8_t *cbp_table; ///< used to store cbp, ac_pred for partitioned decoding uint8_t *pred_dir_table; ///< used to store pred_dir for partitioned decoding - uint8_t *allocated_edge_emu_buffer; - uint8_t *edge_emu_buffer; ///< points into the middle of allocated_edge_emu_buffer + uint8_t *edge_emu_buffer; ///< temporary buffer for if MVs point to out-of-frame data uint8_t *rd_scratchpad; ///< scratchpad for rate distortion mb decision uint8_t *obmc_scratchpad; uint8_t *b_scratchpad; ///< scratchpad used for writing into write only buffers |