avcodec/sheervideo: check build_vlc for failure

Signed-off-by: Paul B Mahol <onemda@gmail.com>

1 files changed, 56 insertions, 51 deletions

diff --git a/libavcodec/sheervideo.c b/libavcodec/sheervideo.c
index 396a46c267..7ae54793a1 100644
--- a/libavcodec/sheervideo.c
+++ b/libavcodec/sheervideo.c
@@ -2893,64 +2893,64 @@ static int decode_frame(AVCodecContext *avctx,
         avctx->pix_fmt = AV_PIX_FMT_RGB0;
         s->decode_frame = decode_rgb;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_r_rgb, 256);
-            build_vlc(&s->vlc[1], l_g_rgb, 256);
+            ret  = build_vlc(&s->vlc[0], l_r_rgb, 256);
+            ret |= build_vlc(&s->vlc[1], l_g_rgb, 256);
         }
         break;
     case MKTAG(' ', 'r', 'G', 'B'):
         avctx->pix_fmt = AV_PIX_FMT_RGB0;
         s->decode_frame = decode_rgbi;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_r_rgbi, 256);
-            build_vlc(&s->vlc[1], l_g_rgbi, 256);
+            ret  = build_vlc(&s->vlc[0], l_r_rgbi, 256);
+            ret |= build_vlc(&s->vlc[1], l_g_rgbi, 256);
         }
         break;
     case MKTAG('A', 'R', 'G', 'X'):
         avctx->pix_fmt = AV_PIX_FMT_GBRAP10;
         s->decode_frame = decode_argx;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_r_rgbx, 1024);
-            build_vlc(&s->vlc[1], l_g_rgbx, 1024);
+            ret  = build_vlc(&s->vlc[0], l_r_rgbx, 1024);
+            ret |= build_vlc(&s->vlc[1], l_g_rgbx, 1024);
         }
         break;
     case MKTAG('A', 'r', 'G', 'X'):
         avctx->pix_fmt = AV_PIX_FMT_GBRAP10;
         s->decode_frame = decode_argxi;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_r_rgbxi, 1024);
-            build_vlc(&s->vlc[1], l_g_rgbxi, 1024);
+            ret  = build_vlc(&s->vlc[0], l_r_rgbxi, 1024);
+            ret |= build_vlc(&s->vlc[1], l_g_rgbxi, 1024);
         }
         break;
     case MKTAG('R', 'G', 'B', 'X'):
         avctx->pix_fmt = AV_PIX_FMT_GBRP10;
         s->decode_frame = decode_rgbx;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_r_rgbx, 1024);
-            build_vlc(&s->vlc[1], l_g_rgbx, 1024);
+            ret  = build_vlc(&s->vlc[0], l_r_rgbx, 1024);
+            ret |= build_vlc(&s->vlc[1], l_g_rgbx, 1024);
         }
         break;
     case MKTAG('r', 'G', 'B', 'X'):
         avctx->pix_fmt = AV_PIX_FMT_GBRP10;
         s->decode_frame = decode_rgbxi;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_r_rgbxi, 1024);
-            build_vlc(&s->vlc[1], l_g_rgbxi, 1024);
+            ret  = build_vlc(&s->vlc[0], l_r_rgbxi, 1024);
+            ret |= build_vlc(&s->vlc[1], l_g_rgbxi, 1024);
         }
         break;
     case MKTAG('A', 'R', 'G', 'B'):
         avctx->pix_fmt = AV_PIX_FMT_ARGB;
         s->decode_frame = decode_argb;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_r_rgb, 256);
-            build_vlc(&s->vlc[1], l_g_rgb, 256);
+            ret  = build_vlc(&s->vlc[0], l_r_rgb, 256);
+            ret |= build_vlc(&s->vlc[1], l_g_rgb, 256);
         }
         break;
     case MKTAG('A', 'r', 'G', 'B'):
         avctx->pix_fmt = AV_PIX_FMT_ARGB;
         s->decode_frame = decode_argbi;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_r_rgbi, 256);
-            build_vlc(&s->vlc[1], l_g_rgbi, 256);
+            ret  = build_vlc(&s->vlc[0], l_r_rgbi, 256);
+            ret |= build_vlc(&s->vlc[1], l_g_rgbi, 256);
         }
         break;
     case MKTAG('A', 'Y', 'B', 'R'):
@@ -2958,8 +2958,8 @@ static int decode_frame(AVCodecContext *avctx,
         avctx->pix_fmt = AV_PIX_FMT_YUVA444P;
         s->decode_frame = decode_aybr;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_ybr, 256);
-            build_vlc(&s->vlc[1], l_u_ybr, 256);
+            ret  = build_vlc(&s->vlc[0], l_y_ybr, 256);
+            ret |= build_vlc(&s->vlc[1], l_u_ybr, 256);
         }
         break;
     case MKTAG('A', 'y', 'B', 'R'):
@@ -2967,8 +2967,8 @@ static int decode_frame(AVCodecContext *avctx,
         avctx->pix_fmt = AV_PIX_FMT_YUVA444P;
         s->decode_frame = decode_aybri;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_ybri, 256);
-            build_vlc(&s->vlc[1], l_u_ybri, 256);
+            ret  = build_vlc(&s->vlc[0], l_y_ybri, 256);
+            ret |= build_vlc(&s->vlc[1], l_u_ybri, 256);
         }
         break;
     case MKTAG(' ', 'Y', 'B', 'R'):
@@ -2976,8 +2976,8 @@ static int decode_frame(AVCodecContext *avctx,
         avctx->pix_fmt = AV_PIX_FMT_YUV444P;
         s->decode_frame = decode_ybr;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_ybr, 256);
-            build_vlc(&s->vlc[1], l_u_ybr, 256);
+            ret  = build_vlc(&s->vlc[0], l_y_ybr, 256);
+            ret |= build_vlc(&s->vlc[1], l_u_ybr, 256);
         }
         break;
     case MKTAG(' ', 'y', 'B', 'R'):
@@ -2985,119 +2985,124 @@ static int decode_frame(AVCodecContext *avctx,
         avctx->pix_fmt = AV_PIX_FMT_YUV444P;
         s->decode_frame = decode_ybri;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_ybri, 256);
-            build_vlc(&s->vlc[1], l_u_ybri, 256);
+            ret  = build_vlc(&s->vlc[0], l_y_ybri, 256);
+            ret |= build_vlc(&s->vlc[1], l_u_ybri, 256);
         }
         break;
     case MKTAG('Y', 'B', 'R', 0x0a):
         avctx->pix_fmt = AV_PIX_FMT_YUV444P10;
         s->decode_frame = decode_ybr10;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_ybr10, 1024);
-            build_vlc(&s->vlc[1], l_u_ybr10, 1024);
+            ret  = build_vlc(&s->vlc[0], l_y_ybr10, 1024);
+            ret |= build_vlc(&s->vlc[1], l_u_ybr10, 1024);
         }
         break;
     case MKTAG('y', 'B', 'R', 0x0a):
         avctx->pix_fmt = AV_PIX_FMT_YUV444P10;
         s->decode_frame = decode_ybr10i;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_ybr10i, 1024);
-            build_vlc(&s->vlc[1], l_u_ybr10i, 1024);
+            ret  = build_vlc(&s->vlc[0], l_y_ybr10i, 1024);
+            ret |= build_vlc(&s->vlc[1], l_u_ybr10i, 1024);
         }
         break;
     case MKTAG('C', 'A', '4', 'p'):
         avctx->pix_fmt = AV_PIX_FMT_YUVA444P10;
         s->decode_frame = decode_ca4p;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_ybr10, 1024);
-            build_vlc(&s->vlc[1], l_u_ybr10, 1024);
+            ret  = build_vlc(&s->vlc[0], l_y_ybr10, 1024);
+            ret |= build_vlc(&s->vlc[1], l_u_ybr10, 1024);
         }
         break;
     case MKTAG('C', 'A', '4', 'i'):
         avctx->pix_fmt = AV_PIX_FMT_YUVA444P10;
         s->decode_frame = decode_ca4i;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_ybr10i, 1024);
-            build_vlc(&s->vlc[1], l_u_ybr10i, 1024);
+            ret  = build_vlc(&s->vlc[0], l_y_ybr10i, 1024);
+            ret |= build_vlc(&s->vlc[1], l_u_ybr10i, 1024);
         }
         break;
     case MKTAG('B', 'Y', 'R', 'Y'):
         avctx->pix_fmt = AV_PIX_FMT_YUV422P;
         s->decode_frame = decode_byry;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_byry, 256);
-            build_vlc(&s->vlc[1], l_u_byry, 256);
+            ret  = build_vlc(&s->vlc[0], l_y_byry, 256);
+            ret |= build_vlc(&s->vlc[1], l_u_byry, 256);
         }
         break;
     case MKTAG('B', 'Y', 'R', 'y'):
         avctx->pix_fmt = AV_PIX_FMT_YUV422P;
         s->decode_frame = decode_byryi;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_byryi, 256);
-            build_vlc(&s->vlc[1], l_u_byryi, 256);
+            ret  = build_vlc(&s->vlc[0], l_y_byryi, 256);
+            ret |= build_vlc(&s->vlc[1], l_u_byryi, 256);
         }
         break;
     case MKTAG('Y', 'b', 'Y', 'r'):
         avctx->pix_fmt = AV_PIX_FMT_YUV422P;
         s->decode_frame = decode_ybyr;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_ybyr, 256);
-            build_vlc(&s->vlc[1], l_u_ybyr, 256);
+            ret  = build_vlc(&s->vlc[0], l_y_ybyr, 256);
+            ret |= build_vlc(&s->vlc[1], l_u_ybyr, 256);
         }
         break;
     case MKTAG('C', '8', '2', 'p'):
         avctx->pix_fmt = AV_PIX_FMT_YUVA422P;
         s->decode_frame = decode_c82p;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_byry, 256);
-            build_vlc(&s->vlc[1], l_u_byry, 256);
+            ret  = build_vlc(&s->vlc[0], l_y_byry, 256);
+            ret |= build_vlc(&s->vlc[1], l_u_byry, 256);
         }
         break;
     case MKTAG('C', '8', '2', 'i'):
         avctx->pix_fmt = AV_PIX_FMT_YUVA422P;
         s->decode_frame = decode_c82i;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_byryi, 256);
-            build_vlc(&s->vlc[1], l_u_byryi, 256);
+            ret  = build_vlc(&s->vlc[0], l_y_byryi, 256);
+            ret |= build_vlc(&s->vlc[1], l_u_byryi, 256);
         }
         break;
     case MKTAG(0xa2, 'Y', 'R', 'Y'):
         avctx->pix_fmt = AV_PIX_FMT_YUV422P10;
         s->decode_frame = decode_yry10;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_yry10, 1024);
-            build_vlc(&s->vlc[1], l_u_yry10, 1024);
+            ret  = build_vlc(&s->vlc[0], l_y_yry10, 1024);
+            ret |= build_vlc(&s->vlc[1], l_u_yry10, 1024);
         }
         break;
     case MKTAG(0xa2, 'Y', 'R', 'y'):
         avctx->pix_fmt = AV_PIX_FMT_YUV422P10;
         s->decode_frame = decode_yry10i;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_yry10i, 1024);
-            build_vlc(&s->vlc[1], l_u_yry10i, 1024);
+            ret  = build_vlc(&s->vlc[0], l_y_yry10i, 1024);
+            ret |= build_vlc(&s->vlc[1], l_u_yry10i, 1024);
         }
         break;
     case MKTAG('C', 'A', '2', 'p'):
         avctx->pix_fmt = AV_PIX_FMT_YUVA422P10;
         s->decode_frame = decode_ca2p;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_yry10, 1024);
-            build_vlc(&s->vlc[1], l_u_yry10, 1024);
+            ret  = build_vlc(&s->vlc[0], l_y_yry10, 1024);
+            ret |= build_vlc(&s->vlc[1], l_u_yry10, 1024);
         }
         break;
     case MKTAG('C', 'A', '2', 'i'):
         avctx->pix_fmt = AV_PIX_FMT_YUVA422P10;
         s->decode_frame = decode_ca2i;
         if (s->format != format) {
-            build_vlc(&s->vlc[0], l_y_yry10i, 1024);
-            build_vlc(&s->vlc[1], l_u_yry10i, 1024);
+            ret  = build_vlc(&s->vlc[0], l_y_yry10i, 1024);
+            ret |= build_vlc(&s->vlc[1], l_u_yry10i, 1024);
         }
         break;
     default:
         avpriv_request_sample(avctx, "unsupported format: 0x%X", format);
         return AVERROR_PATCHWELCOME;
     }
-    s->format = format;
+
+    if (s->format != format) {
+        if (ret < 0)
+            return ret;
+        s->format = format;
+    }
 
     p->pict_type = AV_PICTURE_TYPE_I;
     p->key_frame = 1;