vmdav: unbreak decoding of samples from game The Last Dynasty

This fixes video output with samples HG060808.VMD and
HG060810.VMD. Regression since c0cbe36b18ab3e.

While here show warning if decoding is aborted for some reason.

Signed-off-by: Paul B Mahol <onemda@gmail.com>

1 files changed, 22 insertions, 16 deletions

diff --git a/libavcodec/vmdav.c b/libavcodec/vmdav.c
index f73a8eba36..3e376e301c 100644
--- a/libavcodec/vmdav.c
+++ b/libavcodec/vmdav.c
@@ -192,7 +192,7 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest,
     return bytestream2_tell(&gb);
 }
 
-static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
+static int vmd_decode(VmdVideoContext *s, AVFrame *frame)
 {
     int i;
     unsigned int *palette32;
@@ -213,16 +213,6 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
     frame_y = AV_RL16(&s->buf[8]);
     frame_width = AV_RL16(&s->buf[10]) - frame_x + 1;
     frame_height = AV_RL16(&s->buf[12]) - frame_y + 1;
-    if (frame_x < 0 || frame_width < 0 ||
-        frame_x >= s->avctx->width ||
-        frame_width > s->avctx->width ||
-        frame_x + frame_width > s->avctx->width)
-        return;
-    if (frame_y < 0 || frame_height < 0 ||
-        frame_y >= s->avctx->height ||
-        frame_height > s->avctx->height ||
-        frame_y + frame_height > s->avctx->height)
-        return;
 
     if ((frame_width == s->avctx->width && frame_height == s->avctx->height) &&
         (frame_x || frame_y)) {
@@ -233,6 +223,19 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
     frame_x -= s->x_off;
     frame_y -= s->y_off;
 
+    if (frame_x < 0 || frame_width < 0 ||
+        frame_x >= s->avctx->width ||
+        frame_width > s->avctx->width ||
+        frame_x + frame_width > s->avctx->width) {
+        return AVERROR_INVALIDDATA;
+    }
+    if (frame_y < 0 || frame_height < 0 ||
+        frame_y >= s->avctx->height ||
+        frame_height > s->avctx->height ||
+        frame_y + frame_height > s->avctx->height) {
+        return AVERROR_INVALIDDATA;
+    }
+
     /* if only a certain region will be updated, copy the entire previous
      * frame before the decode */
     if (s->prev_frame.data[0] &&
@@ -262,7 +265,7 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
         /* originally UnpackFrame in VAG's code */
         bytestream2_init(&gb, gb.buffer, s->buf + s->size - gb.buffer);
         if (bytestream2_get_bytes_left(&gb) < 1)
-            return;
+            return AVERROR_INVALIDDATA;
         meth = bytestream2_get_byteu(&gb);
         if (meth & 0x80) {
             lz_unpack(gb.buffer, bytestream2_get_bytes_left(&gb),
@@ -282,13 +285,13 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
                     if (len & 0x80) {
                         len = (len & 0x7F) + 1;
                         if (ofs + len > frame_width || bytestream2_get_bytes_left(&gb) < len)
-                            return;
+                            return AVERROR_INVALIDDATA;
                         bytestream2_get_bufferu(&gb, &dp[ofs], len);
                         ofs += len;
                     } else {
                         /* interframe pixel copy */
                         if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
-                            return;
+                            return AVERROR_INVALIDDATA;
                         memcpy(&dp[ofs], &pp[ofs], len + 1);
                         ofs += len + 1;
                     }
@@ -328,7 +331,7 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
                     } else {
                         /* interframe pixel copy */
                         if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
-                            return;
+                            return AVERROR_INVALIDDATA;
                         memcpy(&dp[ofs], &pp[ofs], len + 1);
                         ofs += len + 1;
                     }
@@ -343,6 +346,8 @@ static void vmd_decode(VmdVideoContext *s, AVFrame *frame)
             break;
         }
     }
+
+    return 0;
 }
 
 static av_cold int vmdvideo_decode_init(AVCodecContext *avctx)
@@ -405,7 +410,8 @@ static int vmdvideo_decode_frame(AVCodecContext *avctx,
     if ((ret = ff_get_buffer(avctx, frame, AV_GET_BUFFER_FLAG_REF)) < 0)
         return ret;
 
-    vmd_decode(s, frame);
+    if (vmd_decode(s, frame) < 0)
+        av_log(avctx, AV_LOG_WARNING, "decode error\n");
 
     /* make the palette available on the way out */
     memcpy(frame->data[1], s->palette, PALETTE_COUNT * 4);