h264: 12 and 14 bit support

This works with I, P and B frames
CAVLC and CABAC

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

3 files changed, 55 insertions, 9 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 0e211a6b36..0d3289458a 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -53,13 +53,17 @@ const uint16_t ff_h264_mb_sizes[4] = { 256, 384, 512, 768 };
 static const uint8_t rem6[QP_MAX_NUM + 1] = {
     0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
     3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
-    0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
+    0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
+    3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
+    0, 1, 2, 3,
 };
 
 static const uint8_t div6[QP_MAX_NUM + 1] = {
     0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3,  3,  3,
     3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6,  6,  6,
-    7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10,
+    7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10,
+   10,10,10,11,11,11,11,11,11,12,12,12,12,12,12,13,13,13, 13, 13, 13,
+   14,14,14,14,
 };
 
 static const enum PixelFormat hwaccel_pixfmt_list_h264_jpeg_420[] = {
@@ -2484,7 +2488,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
 
         if (s->avctx->bits_per_raw_sample != h->sps.bit_depth_luma ||
             h->cur_chroma_format_idc != h->sps.chroma_format_idc) {
-            if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 10 &&
+            if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 14 && h->sps.bit_depth_luma != 11 && h->sps.bit_depth_luma != 13 &&
                 (h->sps.bit_depth_luma != 9 || !CHROMA422)) {
                 s->avctx->bits_per_raw_sample = h->sps.bit_depth_luma;
                 h->cur_chroma_format_idc = h->sps.chroma_format_idc;
@@ -2542,6 +2546,28 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
             else
                 s->avctx->pix_fmt = PIX_FMT_YUV420P10;
             break;
+        case 12:
+            if (CHROMA444) {
+                if (s->avctx->colorspace == AVCOL_SPC_RGB) {
+                    s->avctx->pix_fmt = PIX_FMT_GBRP12;
+                } else
+                    s->avctx->pix_fmt = PIX_FMT_YUV444P12;
+            } else if (CHROMA422)
+                s->avctx->pix_fmt = PIX_FMT_YUV422P12;
+            else
+                s->avctx->pix_fmt = PIX_FMT_YUV420P12;
+            break;
+        case 14:
+            if (CHROMA444) {
+                if (s->avctx->colorspace == AVCOL_SPC_RGB) {
+                    s->avctx->pix_fmt = PIX_FMT_GBRP14;
+                } else
+                    s->avctx->pix_fmt = PIX_FMT_YUV444P14;
+            } else if (CHROMA422)
+                s->avctx->pix_fmt = PIX_FMT_YUV422P14;
+            else
+                s->avctx->pix_fmt = PIX_FMT_YUV420P14;
+            break;
         case 8:
             if (CHROMA444) {
                     s->avctx->pix_fmt = s->avctx->color_range == AVCOL_RANGE_JPEG ? PIX_FMT_YUVJ444P
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 7605e97b57..23dcccf50f 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -104,7 +104,7 @@
  */
 #define DELAYED_PIC_REF 4
 
-#define QP_MAX_NUM (51 + 4*6)           // The maximum supported qp
+#define QP_MAX_NUM (51 + 6*6)           // The maximum supported qp
 
 /* NAL unit types */
 enum {
@@ -598,7 +598,7 @@ typedef struct H264Context {
     int parse_last_mb;
 } H264Context;
 
-extern const uint8_t ff_h264_chroma_qp[5][QP_MAX_NUM + 1]; ///< One chroma qp table for each possible bit depth (8-12).
+extern const uint8_t ff_h264_chroma_qp[7][QP_MAX_NUM + 1]; ///< One chroma qp table for each possible bit depth (8-14).
 extern const uint16_t ff_h264_mb_sizes[4];
 
 /**
diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c
index 1e1794407f..0608a0768d 100644
--- a/libavcodec/h264_ps.c
+++ b/libavcodec/h264_ps.c
@@ -70,7 +70,7 @@ static const AVRational pixel_aspect[17]={
     QP(37,d), QP(37,d), QP(37,d), QP(38,d), QP(38,d), QP(38,d),\
     QP(39,d), QP(39,d), QP(39,d), QP(39,d)
 
-const uint8_t ff_h264_chroma_qp[5][QP_MAX_NUM+1] = {
+const uint8_t ff_h264_chroma_qp[7][QP_MAX_NUM+1] = {
     {
         CHROMA_QP_TABLE_END(8)
     },
@@ -96,6 +96,23 @@ const uint8_t ff_h264_chroma_qp[5][QP_MAX_NUM+1] = {
         18,19,20,21, 22, 23,
         CHROMA_QP_TABLE_END(12)
     },
+    {
+        0,  1, 2, 3,  4,  5,
+        6,  7, 8, 9, 10, 11,
+        12,13,14,15, 16, 17,
+        18,19,20,21, 22, 23,
+        24,25,26,27, 28, 29,
+        CHROMA_QP_TABLE_END(13)
+    },
+    {
+        0,  1, 2, 3,  4,  5,
+        6,  7, 8, 9, 10, 11,
+        12,13,14,15, 16, 17,
+        18,19,20,21, 22, 23,
+        24,25,26,27, 28, 29,
+        30,31,32,33, 34, 35,
+        CHROMA_QP_TABLE_END(14)
+    },
 };
 
 static const uint8_t default_scaling4[2][16]={
@@ -366,7 +383,7 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
         }
         sps->bit_depth_luma   = get_ue_golomb(&s->gb) + 8;
         sps->bit_depth_chroma = get_ue_golomb(&s->gb) + 8;
-        if (sps->bit_depth_luma > 12U || sps->bit_depth_chroma > 12U) {
+        if (sps->bit_depth_luma > 14U || sps->bit_depth_chroma > 14U) {
             av_log(h->s.avctx, AV_LOG_ERROR, "illegal bit depth value (%d, %d)\n",
                    sps->bit_depth_luma, sps->bit_depth_chroma);
             goto fail;
@@ -536,8 +553,11 @@ int ff_h264_decode_picture_parameter_set(H264Context *h, int bit_length){
     if(pps_id >= MAX_PPS_COUNT) {
         av_log(h->s.avctx, AV_LOG_ERROR, "pps_id (%d) out of range\n", pps_id);
         return -1;
-    } else if (h->sps.bit_depth_luma > 10) {
-        av_log(h->s.avctx, AV_LOG_ERROR, "Unimplemented luma bit depth=%d (max=10)\n", h->sps.bit_depth_luma);
+    } else if (h->sps.bit_depth_luma > 14) {
+        av_log(h->s.avctx, AV_LOG_ERROR, "Invalid luma bit depth=%d\n", h->sps.bit_depth_luma);
+        return AVERROR_INVALIDDATA;
+    } else if (h->sps.bit_depth_luma == 11 || h->sps.bit_depth_luma == 13) {
+        av_log(h->s.avctx, AV_LOG_ERROR, "Unimplemented luma bit depth=%d\n", h->sps.bit_depth_luma);
         return AVERROR_PATCHWELCOME;
     }