mpegvideo: move frame size dependent memory management to separate functions

This is a preparation for supporting frame size changes during
frame-based multithreading.

1 files changed, 204 insertions, 173 deletions

diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index f51184f9f6..a1e59af73d 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -654,12 +654,167 @@ void ff_MPV_decode_defaults(MpegEncContext *s)
 }
 
 /**
+ * Initialize and allocates MpegEncContext fields dependent on the resolution.
+ */
+static int init_context_frame(MpegEncContext *s)
+{
+    int y_size, c_size, yc_size, i, mb_array_size, mv_table_size, x, y;
+
+    s->mb_width   = (s->width + 15) / 16;
+    s->mb_stride  = s->mb_width + 1;
+    s->b8_stride  = s->mb_width * 2 + 1;
+    s->b4_stride  = s->mb_width * 4 + 1;
+    mb_array_size = s->mb_height * s->mb_stride;
+    mv_table_size = (s->mb_height + 2) * s->mb_stride + 1;
+
+    /* set default edge pos, will be overriden
+     * in decode_header if needed */
+    s->h_edge_pos = s->mb_width * 16;
+    s->v_edge_pos = s->mb_height * 16;
+
+    s->mb_num     = s->mb_width * s->mb_height;
+
+    s->block_wrap[0] =
+    s->block_wrap[1] =
+    s->block_wrap[2] =
+    s->block_wrap[3] = s->b8_stride;
+    s->block_wrap[4] =
+    s->block_wrap[5] = s->mb_stride;
+
+    y_size  = s->b8_stride * (2 * s->mb_height + 1);
+    c_size  = s->mb_stride * (s->mb_height + 1);
+    yc_size = y_size + 2   * c_size;
+
+    FF_ALLOCZ_OR_GOTO(s->avctx, s->mb_index2xy, (s->mb_num + 1) * sizeof(int),
+                      fail); // error ressilience code looks cleaner with this
+    for (y = 0; y < s->mb_height; y++)
+        for (x = 0; x < s->mb_width; x++)
+            s->mb_index2xy[x + y * s->mb_width] = x + y * s->mb_stride;
+
+    s->mb_index2xy[s->mb_height * s->mb_width] =
+        (s->mb_height - 1) * s->mb_stride + s->mb_width; // FIXME really needed?
+
+    if (s->encoding) {
+        /* Allocate MV tables */
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->p_mv_table_base,
+                          mv_table_size * 2 * sizeof(int16_t), fail);
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->b_forw_mv_table_base,
+                          mv_table_size * 2 * sizeof(int16_t), fail);
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->b_back_mv_table_base,
+                          mv_table_size * 2 * sizeof(int16_t), fail);
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->b_bidir_forw_mv_table_base,
+                          mv_table_size * 2 * sizeof(int16_t), fail);
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->b_bidir_back_mv_table_base,
+                          mv_table_size * 2 * sizeof(int16_t), fail);
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->b_direct_mv_table_base,
+                          mv_table_size * 2 * sizeof(int16_t), fail);
+        s->p_mv_table            = s->p_mv_table_base + s->mb_stride + 1;
+        s->b_forw_mv_table       = s->b_forw_mv_table_base + s->mb_stride + 1;
+        s->b_back_mv_table       = s->b_back_mv_table_base + s->mb_stride + 1;
+        s->b_bidir_forw_mv_table = s->b_bidir_forw_mv_table_base +
+                                   s->mb_stride + 1;
+        s->b_bidir_back_mv_table = s->b_bidir_back_mv_table_base +
+                                   s->mb_stride + 1;
+        s->b_direct_mv_table     = s->b_direct_mv_table_base + s->mb_stride + 1;
+
+        /* Allocate MB type table */
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->mb_type, mb_array_size *
+                          sizeof(uint16_t), fail); // needed for encoding
+
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->lambda_table, mb_array_size *
+                          sizeof(int), fail);
+
+        FF_ALLOC_OR_GOTO(s->avctx, s->cplx_tab,
+                         mb_array_size * sizeof(float), fail);
+        FF_ALLOC_OR_GOTO(s->avctx, s->bits_tab,
+                         mb_array_size * sizeof(float), fail);
+
+    }
+
+    FF_ALLOC_OR_GOTO(s->avctx, s->er_temp_buffer,
+                     mb_array_size * sizeof(uint8_t), fail);
+    FF_ALLOCZ_OR_GOTO(s->avctx, s->error_status_table,
+                      mb_array_size * sizeof(uint8_t), fail);
+
+    if (s->codec_id == AV_CODEC_ID_MPEG4 ||
+        (s->flags & CODEC_FLAG_INTERLACED_ME)) {
+        /* interlaced direct mode decoding tables */
+        for (i = 0; i < 2; i++) {
+            int j, k;
+            for (j = 0; j < 2; j++) {
+                for (k = 0; k < 2; k++) {
+                    FF_ALLOCZ_OR_GOTO(s->avctx,
+                                      s->b_field_mv_table_base[i][j][k],
+                                      mv_table_size * 2 * sizeof(int16_t),
+                                      fail);
+                    s->b_field_mv_table[i][j][k] = s->b_field_mv_table_base[i][j][k] +
+                                                   s->mb_stride + 1;
+                }
+                FF_ALLOCZ_OR_GOTO(s->avctx, s->b_field_select_table [i][j],
+                                  mb_array_size * 2 * sizeof(uint8_t), fail);
+                FF_ALLOCZ_OR_GOTO(s->avctx, s->p_field_mv_table_base[i][j],
+                                  mv_table_size * 2 * sizeof(int16_t), fail);
+                s->p_field_mv_table[i][j] = s->p_field_mv_table_base[i][j]
+                                            + s->mb_stride + 1;
+            }
+            FF_ALLOCZ_OR_GOTO(s->avctx, s->p_field_select_table[i],
+                              mb_array_size * 2 * sizeof(uint8_t), fail);
+        }
+    }
+    if (s->out_format == FMT_H263) {
+        /* cbp values */
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->coded_block_base, y_size, fail);
+        s->coded_block = s->coded_block_base + s->b8_stride + 1;
+
+        /* cbp, ac_pred, pred_dir */
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->cbp_table,
+                          mb_array_size * sizeof(uint8_t), fail);
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->pred_dir_table,
+                          mb_array_size * sizeof(uint8_t), fail);
+    }
+
+    if (s->h263_pred || s->h263_plus || !s->encoding) {
+        /* dc values */
+        // MN: we need these for  error resilience of intra-frames
+        FF_ALLOCZ_OR_GOTO(s->avctx, s->dc_val_base,
+                          yc_size * sizeof(int16_t), fail);
+        s->dc_val[0] = s->dc_val_base + s->b8_stride + 1;
+        s->dc_val[1] = s->dc_val_base + y_size + s->mb_stride + 1;
+        s->dc_val[2] = s->dc_val[1] + c_size;
+        for (i = 0; i < yc_size; i++)
+            s->dc_val_base[i] = 1024;
+    }
+
+    /* which mb is a intra block */
+    FF_ALLOCZ_OR_GOTO(s->avctx, s->mbintra_table, mb_array_size, fail);
+    memset(s->mbintra_table, 1, mb_array_size);
+
+    /* init macroblock skip table */
+    FF_ALLOCZ_OR_GOTO(s->avctx, s->mbskip_table, mb_array_size + 2, fail);
+    // Note the + 1 is for  a quicker mpeg4 slice_end detection
+
+    if ((s->avctx->debug & (FF_DEBUG_VIS_QP | FF_DEBUG_VIS_MB_TYPE)) ||
+        s->avctx->debug_mv) {
+        s->visualization_buffer[0] = av_malloc((s->mb_width * 16 +
+                    2 * EDGE_WIDTH) * s->mb_height * 16 + 2 * EDGE_WIDTH);
+        s->visualization_buffer[1] = av_malloc((s->mb_width * 16 +
+                    2 * EDGE_WIDTH) * s->mb_height * 16 + 2 * EDGE_WIDTH);
+        s->visualization_buffer[2] = av_malloc((s->mb_width * 16 +
+                    2 * EDGE_WIDTH) * s->mb_height * 16 + 2 * EDGE_WIDTH);
+    }
+
+    return 0;
+fail:
+    return AVERROR(ENOMEM);
+}
+
+/**
  * init common structure for both encoder and decoder.
  * this assumes that some variables like width/height are already set
  */
 av_cold int ff_MPV_common_init(MpegEncContext *s)
 {
-    int y_size, c_size, yc_size, i, mb_array_size, mv_table_size, x, y;
+    int i, err;
     int nb_slices = (HAVE_THREADS &&
                      s->avctx->active_thread_type & FF_THREAD_SLICE) ?
                     s->avctx->thread_count : 1;
@@ -699,35 +854,10 @@ av_cold int ff_MPV_common_init(MpegEncContext *s)
     s->flags2 = s->avctx->flags2;
 
     if (s->width && s->height) {
-        s->mb_width   = (s->width + 15) / 16;
-        s->mb_stride  = s->mb_width + 1;
-        s->b8_stride  = s->mb_width * 2 + 1;
-        s->b4_stride  = s->mb_width * 4 + 1;
-        mb_array_size = s->mb_height * s->mb_stride;
-        mv_table_size = (s->mb_height + 2) * s->mb_stride + 1;
-
         /* set chroma shifts */
         avcodec_get_chroma_sub_sample(s->avctx->pix_fmt, &s->chroma_x_shift,
                                       &s->chroma_y_shift);
 
-        /* set default edge pos, will be overriden
-         * in decode_header if needed */
-        s->h_edge_pos = s->mb_width * 16;
-        s->v_edge_pos = s->mb_height * 16;
-
-        s->mb_num     = s->mb_width * s->mb_height;
-
-        s->block_wrap[0] =
-        s->block_wrap[1] =
-        s->block_wrap[2] =
-        s->block_wrap[3] = s->b8_stride;
-        s->block_wrap[4] =
-        s->block_wrap[5] = s->mb_stride;
-
-        y_size  = s->b8_stride * (2 * s->mb_height + 1);
-        c_size  = s->mb_stride * (s->mb_height + 1);
-        yc_size = y_size + 2   * c_size;
-
         /* convert fourcc to upper case */
         s->codec_tag          = avpriv_toupper4(s->avctx->codec_tag);
 
@@ -735,42 +865,7 @@ av_cold int ff_MPV_common_init(MpegEncContext *s)
 
         s->avctx->coded_frame = &s->current_picture.f;
 
-        FF_ALLOCZ_OR_GOTO(s->avctx, s->mb_index2xy, (s->mb_num + 1) * sizeof(int),
-                          fail); // error ressilience code looks cleaner with this
-        for (y = 0; y < s->mb_height; y++)
-            for (x = 0; x < s->mb_width; x++)
-                s->mb_index2xy[x + y * s->mb_width] = x + y * s->mb_stride;
-
-        s->mb_index2xy[s->mb_height * s->mb_width] =
-                       (s->mb_height - 1) * s->mb_stride + s->mb_width; // FIXME really needed?
-
         if (s->encoding) {
-            /* Allocate MV tables */
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->p_mv_table_base,
-                              mv_table_size * 2 * sizeof(int16_t), fail);
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->b_forw_mv_table_base,
-                              mv_table_size * 2 * sizeof(int16_t), fail);
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->b_back_mv_table_base,
-                              mv_table_size * 2 * sizeof(int16_t), fail);
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->b_bidir_forw_mv_table_base,
-                              mv_table_size * 2 * sizeof(int16_t), fail);
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->b_bidir_back_mv_table_base,
-                              mv_table_size * 2 * sizeof(int16_t), fail);
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->b_direct_mv_table_base,
-                              mv_table_size * 2 * sizeof(int16_t), fail);
-            s->p_mv_table            = s->p_mv_table_base +
-                                       s->mb_stride + 1;
-            s->b_forw_mv_table       = s->b_forw_mv_table_base +
-                                       s->mb_stride + 1;
-            s->b_back_mv_table       = s->b_back_mv_table_base +
-                                       s->mb_stride + 1;
-            s->b_bidir_forw_mv_table = s->b_bidir_forw_mv_table_base +
-                                       s->mb_stride + 1;
-            s->b_bidir_back_mv_table = s->b_bidir_back_mv_table_base +
-                                       s->mb_stride + 1;
-            s->b_direct_mv_table     = s->b_direct_mv_table_base +
-                                       s->mb_stride + 1;
-
             if (s->msmpeg4_version) {
                 FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
                                   2 * 2 * (MAX_LEVEL + 1) *
@@ -778,13 +873,6 @@ av_cold int ff_MPV_common_init(MpegEncContext *s)
             }
             FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
 
-            /* Allocate MB type table */
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->mb_type, mb_array_size *
-                              sizeof(uint16_t), fail); // needed for encoding
-
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->lambda_table, mb_array_size *
-                              sizeof(int), fail);
-
             FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,
                               64 * 32   * sizeof(int), fail);
             FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,
@@ -802,11 +890,6 @@ av_cold int ff_MPV_common_init(MpegEncContext *s)
                 FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
                                   2 * 64 * sizeof(uint16_t), fail);
             }
-
-            FF_ALLOC_OR_GOTO(s->avctx, s->cplx_tab,
-                             mb_array_size * sizeof(float), fail);
-            FF_ALLOC_OR_GOTO(s->avctx, s->bits_tab,
-                             mb_array_size * sizeof(float), fail);
         }
     }
 
@@ -818,81 +901,10 @@ av_cold int ff_MPV_common_init(MpegEncContext *s)
     }
 
     if (s->width && s->height) {
-        FF_ALLOC_OR_GOTO(s->avctx, s->er_temp_buffer,
-                         mb_array_size * sizeof(uint8_t), fail);
-        FF_ALLOCZ_OR_GOTO(s->avctx, s->error_status_table,
-                          mb_array_size * sizeof(uint8_t), fail);
-
-        if (s->codec_id == AV_CODEC_ID_MPEG4 ||
-            (s->flags & CODEC_FLAG_INTERLACED_ME)) {
-            /* interlaced direct mode decoding tables */
-            for (i = 0; i < 2; i++) {
-                int j, k;
-                for (j = 0; j < 2; j++) {
-                    for (k = 0; k < 2; k++) {
-                        FF_ALLOCZ_OR_GOTO(s->avctx,
-                                          s->b_field_mv_table_base[i][j][k],
-                                          mv_table_size * 2 * sizeof(int16_t),
-                                          fail);
-                        s->b_field_mv_table[i][j][k] = s->b_field_mv_table_base[i][j][k] +
-                                                       s->mb_stride + 1;
-                    }
-                    FF_ALLOCZ_OR_GOTO(s->avctx, s->b_field_select_table [i][j],
-                                      mb_array_size * 2 * sizeof(uint8_t),
-                                      fail);
-                    FF_ALLOCZ_OR_GOTO(s->avctx, s->p_field_mv_table_base[i][j],
-                                      mv_table_size * 2 * sizeof(int16_t),
-                                      fail);
-                    s->p_field_mv_table[i][j] = s->p_field_mv_table_base[i][j]
-                                                + s->mb_stride + 1;
-                }
-                FF_ALLOCZ_OR_GOTO(s->avctx, s->p_field_select_table[i],
-                                  mb_array_size * 2 * sizeof(uint8_t),
-                                  fail);
-            }
-        }
-        if (s->out_format == FMT_H263) {
-            /* cbp values */
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->coded_block_base, y_size, fail);
-            s->coded_block = s->coded_block_base + s->b8_stride + 1;
-
-            /* cbp, ac_pred, pred_dir */
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->cbp_table,
-                              mb_array_size * sizeof(uint8_t), fail);
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->pred_dir_table,
-                              mb_array_size * sizeof(uint8_t), fail);
-        }
-
-        if (s->h263_pred || s->h263_plus || !s->encoding) {
-            /* dc values */
-            // MN: we need these for  error resilience of intra-frames
-            FF_ALLOCZ_OR_GOTO(s->avctx, s->dc_val_base,
-                              yc_size * sizeof(int16_t), fail);
-            s->dc_val[0] = s->dc_val_base + s->b8_stride + 1;
-            s->dc_val[1] = s->dc_val_base + y_size + s->mb_stride + 1;
-            s->dc_val[2] = s->dc_val[1] + c_size;
-            for (i = 0; i < yc_size; i++)
-                s->dc_val_base[i] = 1024;
-        }
-
-        /* which mb is a intra block */
-        FF_ALLOCZ_OR_GOTO(s->avctx, s->mbintra_table, mb_array_size, fail);
-        memset(s->mbintra_table, 1, mb_array_size);
-
-        /* init macroblock skip table */
-        FF_ALLOCZ_OR_GOTO(s->avctx, s->mbskip_table, mb_array_size + 2, fail);
-        // Note the + 1 is for  a quicker mpeg4 slice_end detection
+        if ((err = init_context_frame(s)))
+            goto fail;
 
         s->parse_context.state = -1;
-        if ((s->avctx->debug & (FF_DEBUG_VIS_QP | FF_DEBUG_VIS_MB_TYPE)) ||
-            s->avctx->debug_mv) {
-            s->visualization_buffer[0] = av_malloc((s->mb_width * 16 +
-                        2 * EDGE_WIDTH) * s->mb_height * 16 + 2 * EDGE_WIDTH);
-            s->visualization_buffer[1] = av_malloc((s->mb_width * 16 +
-                        2 * EDGE_WIDTH) * s->mb_height * 16 + 2 * EDGE_WIDTH);
-            s->visualization_buffer[2] = av_malloc((s->mb_width * 16 +
-                        2 * EDGE_WIDTH) * s->mb_height * 16 + 2 * EDGE_WIDTH);
-        }
     }
 
     s->context_initialized = 1;
@@ -928,24 +940,15 @@ av_cold int ff_MPV_common_init(MpegEncContext *s)
     return -1;
 }
 
-/* init common structure for both encoder and decoder */
-void ff_MPV_common_end(MpegEncContext *s)
+/**
+ * Frees and resets MpegEncContext fields depending on the resolution.
+ * Is used during resolution changes to avoid a full reinitialization of the
+ * codec.
+ */
+static int free_context_frame(MpegEncContext *s)
 {
     int i, j, k;
 
-    if (s->slice_context_count > 1) {
-        for (i = 0; i < s->slice_context_count; i++) {
-            free_duplicate_context(s->thread_context[i]);
-        }
-        for (i = 1; i < s->slice_context_count; i++) {
-            av_freep(&s->thread_context[i]);
-        }
-        s->slice_context_count = 1;
-    } else free_duplicate_context(s);
-
-    av_freep(&s->parse_context.buffer);
-    s->parse_context.buffer_size = 0;
-
     av_freep(&s->mb_type);
     av_freep(&s->p_mv_table_base);
     av_freep(&s->b_forw_mv_table_base);
@@ -979,15 +982,49 @@ void ff_MPV_common_end(MpegEncContext *s)
     av_freep(&s->pred_dir_table);
 
     av_freep(&s->mbskip_table);
-    av_freep(&s->bitstream_buffer);
-    s->allocated_bitstream_buffer_size = 0;
 
-    av_freep(&s->avctx->stats_out);
-    av_freep(&s->ac_stats);
     av_freep(&s->error_status_table);
     av_freep(&s->er_temp_buffer);
     av_freep(&s->mb_index2xy);
     av_freep(&s->lambda_table);
+    av_freep(&s->cplx_tab);
+    av_freep(&s->bits_tab);
+
+    s->linesize = s->uvlinesize = 0;
+
+    for (i = 0; i < 3; i++)
+        av_freep(&s->visualization_buffer[i]);
+
+    if (!(s->avctx->active_thread_type & FF_THREAD_FRAME))
+        avcodec_default_free_buffers(s->avctx);
+
+    return 0;
+}
+
+/* init common structure for both encoder and decoder */
+void ff_MPV_common_end(MpegEncContext *s)
+{
+    int i;
+
+    if (s->slice_context_count > 1) {
+        for (i = 0; i < s->slice_context_count; i++) {
+            free_duplicate_context(s->thread_context[i]);
+        }
+        for (i = 1; i < s->slice_context_count; i++) {
+            av_freep(&s->thread_context[i]);
+        }
+        s->slice_context_count = 1;
+    } else free_duplicate_context(s);
+
+    av_freep(&s->parse_context.buffer);
+    s->parse_context.buffer_size = 0;
+
+    av_freep(&s->bitstream_buffer);
+    s->allocated_bitstream_buffer_size = 0;
+
+    av_freep(&s->avctx->stats_out);
+    av_freep(&s->ac_stats);
+
     av_freep(&s->q_intra_matrix);
     av_freep(&s->q_inter_matrix);
     av_freep(&s->q_intra_matrix16);
@@ -995,8 +1032,8 @@ void ff_MPV_common_end(MpegEncContext *s)
     av_freep(&s->input_picture);
     av_freep(&s->reordered_input_picture);
     av_freep(&s->dct_offset);
-    av_freep(&s->cplx_tab);
-    av_freep(&s->bits_tab);
+
+    free_context_frame(s);
 
     if (s->picture && !s->avctx->internal->is_copy) {
         for (i = 0; i < s->picture_count; i++) {
@@ -1009,12 +1046,6 @@ void ff_MPV_common_end(MpegEncContext *s)
     s->next_picture_ptr         =
     s->current_picture_ptr      = NULL;
     s->linesize = s->uvlinesize = 0;
-
-    for (i = 0; i < 3; i++)
-        av_freep(&s->visualization_buffer[i]);
-
-    if (!(s->avctx->active_thread_type & FF_THREAD_FRAME))
-        avcodec_default_free_buffers(s->avctx);
 }
 
 void ff_init_rl(RLTable *rl,