diff options
| author | Andreas Rheinhardt <andreas.rheinhardt@outlook.com> | 2021-09-16 14:14:24 +0200 |
|---|---|---|
| committer | Andreas Rheinhardt <andreas.rheinhardt@outlook.com> | 2021-09-24 00:00:13 +0200 |
| commit | ad27326e2b51ba2566bd7b72bea8e8031c2bc4c5 (patch) | |
| tree | 72e025d3df70b43b0c95d45610c52a0c918a1cb8 /libavcodec | |
| parent | 477a398c3ea9b84cb3c8139becadbb821e234ceb (diff) | |
avcodec/cinepakenc: Check all calls to avpriv_elbg_do()
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Diffstat (limited to 'libavcodec')
| -rw-r--r-- | libavcodec/cinepakenc.c | 28 |
1 files changed, 22 insertions, 6 deletions
diff --git a/libavcodec/cinepakenc.c b/libavcodec/cinepakenc.c index 2984b93de3..d1bcf2b2d5 100644 --- a/libavcodec/cinepakenc.c +++ b/libavcodec/cinepakenc.c @@ -709,6 +709,7 @@ static int quantize(CinepakEncContext *s, int h, uint8_t *data[4], uint8_t vq_pict_buf[(MB_AREA * 3) / 2]; uint8_t *sub_data[4], *vq_data[4]; int sub_linesize[4], vq_linesize[4]; + int ret; for (mbn = i = y = 0; y < h; y += MB_SIZE) { for (x = 0; x < s->w; x += MB_SIZE, ++mbn) { @@ -762,8 +763,10 @@ static int quantize(CinepakEncContext *s, int h, uint8_t *data[4], if (i < size) size = i; - avpriv_elbg_do(&s->elbg, s->codebook_input, entry_size, i, codebook, - size, 1, s->codebook_closest, &s->randctx); + ret = avpriv_elbg_do(&s->elbg, s->codebook_input, entry_size, i, codebook, + size, 1, s->codebook_closest, &s->randctx); + if (ret < 0) + return ret; // set up vq_data, which contains a single MB vq_data[0] = vq_pict_buf; @@ -888,8 +891,10 @@ static int rd_strip(CinepakEncContext *s, int y, int h, int keyframe, if (mode == MODE_V1_ONLY) { info.v1_size = v1_size; // the size may shrink even before optimizations if the input is short: - info.v1_size = quantize(s, h, data, linesize, 1, - &info, ENC_UNCERTAIN); + if ((new_v1_size = quantize(s, h, data, linesize, 1, + &info, ENC_UNCERTAIN)) < 0) + return new_v1_size; + info.v1_size = new_v1_size; if (info.v1_size < v1_size) // too few eligible blocks, no sense in trying bigger sizes v1enough = 1; @@ -902,8 +907,11 @@ static int rd_strip(CinepakEncContext *s, int y, int h, int keyframe, if (mode == MODE_V1_V4) { info.v4_size = v4_size; - info.v4_size = quantize(s, h, data, linesize, 0, - &info, ENC_UNCERTAIN); + new_v4_size = quantize(s, h, data, linesize, 0, + &info, ENC_UNCERTAIN); + if (new_v4_size < 0) + return new_v4_size; + info.v4_size = new_v4_size; if (info.v4_size < v4_size) // too few eligible blocks, no sense in trying bigger sizes v4enough = 1; @@ -921,11 +929,15 @@ static int rd_strip(CinepakEncContext *s, int y, int h, int keyframe, // we assume we _may_ come here with more blocks to encode than before info.v1_size = v1_size; new_v1_size = quantize(s, h, data, linesize, 1, &info, ENC_V1); + if (new_v1_size < 0) + return new_v1_size; if (new_v1_size < info.v1_size) info.v1_size = new_v1_size; // we assume we _may_ come here with more blocks to encode than before info.v4_size = v4_size; new_v4_size = quantize(s, h, data, linesize, 0, &info, ENC_V4); + if (new_v4_size < 0) + return new_v4_size; if (new_v4_size < info.v4_size) info.v4_size = new_v4_size; // calculate the resulting score @@ -942,12 +954,16 @@ static int rd_strip(CinepakEncContext *s, int y, int h, int keyframe, if (v1shrunk) { info.v1_size = v1_size; new_v1_size = quantize(s, h, data, linesize, 1, &info, ENC_V1); + if (new_v1_size < 0) + return new_v1_size; if (new_v1_size < info.v1_size) info.v1_size = new_v1_size; } if (v4shrunk) { info.v4_size = v4_size; new_v4_size = quantize(s, h, data, linesize, 0, &info, ENC_V4); + if (new_v4_size < 0) + return new_v4_size; if (new_v4_size < info.v4_size) info.v4_size = new_v4_size; } |