h264: move ff_h264_check_intra[4x4]_pred_mode() to h264_parse

It is shared with svq3.

8 files changed, 128 insertions, 116 deletions

diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 54b12d35d3..e66ec51f4e 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -418,7 +418,7 @@ OBJS-$(CONFIG_SUNRAST_ENCODER)         += sunrastenc.o
 OBJS-$(CONFIG_SVQ1_DECODER)            += svq1dec.o svq1.o svq13.o h263data.o
 OBJS-$(CONFIG_SVQ1_ENCODER)            += svq1enc.o svq1.o  h263data.o  \
                                           h263.o ituh263enc.o
-OBJS-$(CONFIG_SVQ3_DECODER)            += svq3.o svq13.o mpegutils.o
+OBJS-$(CONFIG_SVQ3_DECODER)            += svq3.o svq13.o mpegutils.o h264_parse.o
 OBJS-$(CONFIG_TAK_DECODER)             += takdec.o tak.o
 OBJS-$(CONFIG_TARGA_DECODER)           += targa.o
 OBJS-$(CONFIG_TARGA_ENCODER)           += targaenc.o rle.o
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 49b3f93842..a90ef6477d 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -114,99 +114,6 @@ void ff_h264_draw_horiz_band(const H264Context *h, H264SliceContext *sl,
     }
 }
 
-/**
- * Check if the top & left blocks are available if needed and
- * change the dc mode so it only uses the available blocks.
- */
-int ff_h264_check_intra4x4_pred_mode(const H264Context *h, H264SliceContext *sl)
-{
-    static const int8_t top[12] = {
-        -1, 0, LEFT_DC_PRED, -1, -1, -1, -1, -1, 0
-    };
-    static const int8_t left[12] = {
-        0, -1, TOP_DC_PRED, 0, -1, -1, -1, 0, -1, DC_128_PRED
-    };
-    int i;
-
-    if (!(sl->top_samples_available & 0x8000)) {
-        for (i = 0; i < 4; i++) {
-            int status = top[sl->intra4x4_pred_mode_cache[scan8[0] + i]];
-            if (status < 0) {
-                av_log(h->avctx, AV_LOG_ERROR,
-                       "top block unavailable for requested intra4x4 mode %d at %d %d\n",
-                       status, sl->mb_x, sl->mb_y);
-                return AVERROR_INVALIDDATA;
-            } else if (status) {
-                sl->intra4x4_pred_mode_cache[scan8[0] + i] = status;
-            }
-        }
-    }
-
-    if ((sl->left_samples_available & 0x8888) != 0x8888) {
-        static const int mask[4] = { 0x8000, 0x2000, 0x80, 0x20 };
-        for (i = 0; i < 4; i++)
-            if (!(sl->left_samples_available & mask[i])) {
-                int status = left[sl->intra4x4_pred_mode_cache[scan8[0] + 8 * i]];
-                if (status < 0) {
-                    av_log(h->avctx, AV_LOG_ERROR,
-                           "left block unavailable for requested intra4x4 mode %d at %d %d\n",
-                           status, sl->mb_x, sl->mb_y);
-                    return AVERROR_INVALIDDATA;
-                } else if (status) {
-                    sl->intra4x4_pred_mode_cache[scan8[0] + 8 * i] = status;
-                }
-            }
-    }
-
-    return 0;
-} // FIXME cleanup like ff_h264_check_intra_pred_mode
-
-/**
- * Check if the top & left blocks are available if needed and
- * change the dc mode so it only uses the available blocks.
- */
-int ff_h264_check_intra_pred_mode(const H264Context *h, H264SliceContext *sl,
-                                  int mode, int is_chroma)
-{
-    static const int8_t top[4]  = { LEFT_DC_PRED8x8, 1, -1, -1 };
-    static const int8_t left[5] = { TOP_DC_PRED8x8, -1,  2, -1, DC_128_PRED8x8 };
-
-    if (mode > 3U) {
-        av_log(h->avctx, AV_LOG_ERROR,
-               "out of range intra chroma pred mode at %d %d\n",
-               sl->mb_x, sl->mb_y);
-        return AVERROR_INVALIDDATA;
-    }
-
-    if (!(sl->top_samples_available & 0x8000)) {
-        mode = top[mode];
-        if (mode < 0) {
-            av_log(h->avctx, AV_LOG_ERROR,
-                   "top block unavailable for requested intra mode at %d %d\n",
-                   sl->mb_x, sl->mb_y);
-            return AVERROR_INVALIDDATA;
-        }
-    }
-
-    if ((sl->left_samples_available & 0x8080) != 0x8080) {
-        mode = left[mode];
-        if (is_chroma && (sl->left_samples_available & 0x8080)) {
-            // mad cow disease mode, aka MBAFF + constrained_intra_pred
-            mode = ALZHEIMER_DC_L0T_PRED8x8 +
-                   (!(sl->left_samples_available & 0x8000)) +
-                   2 * (mode == DC_128_PRED8x8);
-        }
-        if (mode < 0) {
-            av_log(h->avctx, AV_LOG_ERROR,
-                   "left block unavailable for requested intra mode at %d %d\n",
-                   sl->mb_x, sl->mb_y);
-            return AVERROR_INVALIDDATA;
-        }
-    }
-
-    return mode;
-}
-
 const uint8_t *ff_h264_decode_nal(H264Context *h, H264SliceContext *sl,
                                   const uint8_t *src,
                                   int *dst_length, int *consumed, int length)
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 3fe5b9c7e5..e00ce8857f 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -808,19 +808,6 @@ int ff_h264_decode_ref_pic_marking(H264Context *h, GetBitContext *gb,
 
 int ff_generate_sliding_window_mmcos(H264Context *h, int first_slice);
 
-/**
- * Check if the top & left blocks are available if needed & change the
- * dc mode so it only uses the available blocks.
- */
-int ff_h264_check_intra4x4_pred_mode(const H264Context *h, H264SliceContext *sl);
-
-/**
- * Check if the top & left blocks are available if needed & change the
- * dc mode so it only uses the available blocks.
- */
-int ff_h264_check_intra_pred_mode(const H264Context *h, H264SliceContext *sl,
-                                  int mode, int is_chroma);
-
 void ff_h264_hl_decode_mb(const H264Context *h, H264SliceContext *sl);
 int ff_h264_decode_extradata(H264Context *h);
 int ff_h264_decode_init(AVCodecContext *avctx);
diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c
index 37f98ddd8a..589a021e1f 100644
--- a/libavcodec/h264_cabac.c
+++ b/libavcodec/h264_cabac.c
@@ -2076,16 +2076,20 @@ decode_intra_mb:
                 }
             }
             write_back_intra_pred_mode(h, sl);
-            if (ff_h264_check_intra4x4_pred_mode(h, sl) < 0 ) return -1;
+            if (ff_h264_check_intra4x4_pred_mode(sl->intra4x4_pred_mode_cache, h->avctx,
+                                                 sl->top_samples_available, sl->left_samples_available) < 0 )
+                return -1;
         } else {
-            sl->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, sl, sl->intra16x16_pred_mode, 0);
+            sl->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h->avctx, sl->top_samples_available,
+                                                                     sl->left_samples_available, sl->intra16x16_pred_mode, 0);
             if (sl->intra16x16_pred_mode < 0) return -1;
         }
         if(decode_chroma){
             h->chroma_pred_mode_table[mb_xy] =
             pred_mode                        = decode_cabac_mb_chroma_pre_mode(h, sl);
 
-            pred_mode= ff_h264_check_intra_pred_mode(h, sl, pred_mode, 1 );
+            pred_mode= ff_h264_check_intra_pred_mode(h->avctx, sl->top_samples_available,
+                                                     sl->left_samples_available, pred_mode, 1 );
             if( pred_mode < 0 ) return -1;
             sl->chroma_pred_mode = pred_mode;
         } else {
diff --git a/libavcodec/h264_cavlc.c b/libavcodec/h264_cavlc.c
index 45a838603e..bdd9f73975 100644
--- a/libavcodec/h264_cavlc.c
+++ b/libavcodec/h264_cavlc.c
@@ -818,15 +818,18 @@ decode_intra_mb:
                     sl->intra4x4_pred_mode_cache[scan8[i]] = mode;
             }
             write_back_intra_pred_mode(h, sl);
-            if (ff_h264_check_intra4x4_pred_mode(h, sl) < 0)
+            if (ff_h264_check_intra4x4_pred_mode(sl->intra4x4_pred_mode_cache, h->avctx,
+                                                 sl->top_samples_available, sl->left_samples_available) < 0)
                 return -1;
         }else{
-            sl->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, sl, sl->intra16x16_pred_mode, 0);
+            sl->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h->avctx, sl->top_samples_available,
+                                                                     sl->left_samples_available, sl->intra16x16_pred_mode, 0);
             if (sl->intra16x16_pred_mode < 0)
                 return -1;
         }
         if(decode_chroma){
-            pred_mode= ff_h264_check_intra_pred_mode(h, sl, get_ue_golomb_31(&sl->gb), 1);
+            pred_mode= ff_h264_check_intra_pred_mode(h->avctx, sl->top_samples_available,
+                                                     sl->left_samples_available, get_ue_golomb_31(&sl->gb), 1);
             if(pred_mode < 0)
                 return -1;
             sl->chroma_pred_mode = pred_mode;
diff --git a/libavcodec/h264_parse.c b/libavcodec/h264_parse.c
index 88caa48f19..50b111820f 100644
--- a/libavcodec/h264_parse.c
+++ b/libavcodec/h264_parse.c
@@ -84,3 +84,95 @@ int ff_h264_pred_weight_table(GetBitContext *gb, const SPS *sps,
     pwt->use_weight = pwt->use_weight || pwt->use_weight_chroma;
     return 0;
 }
+
+/**
+ * Check if the top & left blocks are available if needed and
+ * change the dc mode so it only uses the available blocks.
+ */
+int ff_h264_check_intra4x4_pred_mode(int8_t *pred_mode_cache, void *logctx,
+                                     int top_samples_available, int left_samples_available)
+{
+    static const int8_t top[12] = {
+        -1, 0, LEFT_DC_PRED, -1, -1, -1, -1, -1, 0
+    };
+    static const int8_t left[12] = {
+        0, -1, TOP_DC_PRED, 0, -1, -1, -1, 0, -1, DC_128_PRED
+    };
+    int i;
+
+    if (!(top_samples_available & 0x8000)) {
+        for (i = 0; i < 4; i++) {
+            int status = top[pred_mode_cache[scan8[0] + i]];
+            if (status < 0) {
+                av_log(logctx, AV_LOG_ERROR,
+                       "top block unavailable for requested intra4x4 mode %d\n",
+                       status);
+                return AVERROR_INVALIDDATA;
+            } else if (status) {
+                pred_mode_cache[scan8[0] + i] = status;
+            }
+        }
+    }
+
+    if ((left_samples_available & 0x8888) != 0x8888) {
+        static const int mask[4] = { 0x8000, 0x2000, 0x80, 0x20 };
+        for (i = 0; i < 4; i++)
+            if (!(left_samples_available & mask[i])) {
+                int status = left[pred_mode_cache[scan8[0] + 8 * i]];
+                if (status < 0) {
+                    av_log(logctx, AV_LOG_ERROR,
+                           "left block unavailable for requested intra4x4 mode %d\n",
+                           status);
+                    return AVERROR_INVALIDDATA;
+                } else if (status) {
+                    pred_mode_cache[scan8[0] + 8 * i] = status;
+                }
+            }
+    }
+
+    return 0;
+}
+
+/**
+ * Check if the top & left blocks are available if needed and
+ * change the dc mode so it only uses the available blocks.
+ */
+int ff_h264_check_intra_pred_mode(void *logctx, int top_samples_available,
+                                  int left_samples_available,
+                                  int mode, int is_chroma)
+{
+    static const int8_t top[4]  = { LEFT_DC_PRED8x8, 1, -1, -1 };
+    static const int8_t left[5] = { TOP_DC_PRED8x8, -1,  2, -1, DC_128_PRED8x8 };
+
+    if (mode > 3U) {
+        av_log(logctx, AV_LOG_ERROR,
+               "out of range intra chroma pred mode\n");
+        return AVERROR_INVALIDDATA;
+    }
+
+    if (!(top_samples_available & 0x8000)) {
+        mode = top[mode];
+        if (mode < 0) {
+            av_log(logctx, AV_LOG_ERROR,
+                   "top block unavailable for requested intra mode\n");
+            return AVERROR_INVALIDDATA;
+        }
+    }
+
+    if ((left_samples_available & 0x8080) != 0x8080) {
+        mode = left[mode];
+        if (is_chroma && (left_samples_available & 0x8080)) {
+            // mad cow disease mode, aka MBAFF + constrained_intra_pred
+            mode = ALZHEIMER_DC_L0T_PRED8x8 +
+                   (!(left_samples_available & 0x8000)) +
+                   2 * (mode == DC_128_PRED8x8);
+        }
+        if (mode < 0) {
+            av_log(logctx, AV_LOG_ERROR,
+                   "left block unavailable for requested intra mode\n");
+            return AVERROR_INVALIDDATA;
+        }
+    }
+
+    return mode;
+}
diff --git a/libavcodec/h264_parse.h b/libavcodec/h264_parse.h
index 7bdce04a3b..9b96dc47a0 100644
--- a/libavcodec/h264_parse.h
+++ b/libavcodec/h264_parse.h
@@ -45,4 +45,19 @@ int ff_h264_pred_weight_table(GetBitContext *gb, const struct SPS *sps,
                               const int *ref_count, int slice_type_nos,
                               H264PredWeightTable *pwt);
 
+/**
+ * Check if the top & left blocks are available if needed & change the
+ * dc mode so it only uses the available blocks.
+ */
+int ff_h264_check_intra4x4_pred_mode(int8_t *pred_mode_cache, void *logctx,
+                                     int top_samples_available, int left_samples_available);
+
+/**
+ * Check if the top & left blocks are available if needed & change the
+ * dc mode so it only uses the available blocks.
+ */
+int ff_h264_check_intra_pred_mode(void *logctx, int top_samples_available,
+                                  int left_samples_available,
+                                  int mode, int is_chroma);
+
 #endif /* AVCODEC_H264_PARSE_H */
diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c
index 2bb957d94c..0d96abd157 100644
--- a/libavcodec/svq3.c
+++ b/libavcodec/svq3.c
@@ -771,7 +771,9 @@ static int svq3_decode_mb(SVQ3Context *s, unsigned int mb_type)
         i4x4[6] = i4x4_cache[7 + 8 * 1];
 
         if (mb_type == 8) {
-            ff_h264_check_intra4x4_pred_mode(h, sl);
+            ff_h264_check_intra4x4_pred_mode(sl->intra4x4_pred_mode_cache,
+                                             h->avctx, sl->top_samples_available,
+                                             sl->left_samples_available);
 
             sl->top_samples_available  = (s->mb_y == 0) ? 0x33FF : 0xFFFF;
             sl->left_samples_available = (s->mb_x == 0) ? 0x5F5F : 0xFFFF;
@@ -788,7 +790,8 @@ static int svq3_decode_mb(SVQ3Context *s, unsigned int mb_type)
         dir = ff_h264_i_mb_type_info[mb_type - 8].pred_mode;
         dir = (dir >> 1) ^ 3 * (dir & 1) ^ 1;
 
-        if ((sl->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, sl, dir, 0)) < 0) {
+        if ((sl->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h->avctx, sl->top_samples_available,
+                                                                      sl->left_samples_available, dir, 0)) < 0) {
             av_log(h->avctx, AV_LOG_ERROR, "ff_h264_check_intra_pred_mode < 0\n");
             return sl->intra16x16_pred_mode;
         }
@@ -892,7 +895,8 @@ static int svq3_decode_mb(SVQ3Context *s, unsigned int mb_type)
     h->cur_pic.mb_type[mb_xy] = mb_type;
 
     if (IS_INTRA(mb_type))
-        sl->chroma_pred_mode = ff_h264_check_intra_pred_mode(h, sl, DC_PRED8x8, 1);
+        sl->chroma_pred_mode = ff_h264_check_intra_pred_mode(h->avctx, sl->top_samples_available,
+                                                             sl->left_samples_available, DC_PRED8x8, 1);
 
     return 0;
 }