avcodec/cfhdenc: extend quantization table

Should give better quality.

1 files changed, 81 insertions, 16 deletions

diff --git a/libavcodec/cfhdenc.c b/libavcodec/cfhdenc.c
index 68644529da..911eee4e3e 100644
--- a/libavcodec/cfhdenc.c
+++ b/libavcodec/cfhdenc.c
@@ -103,17 +103,85 @@ static const uint16_t runbook[8][3] = {
  * Derived by inspecting various quality encodes
  * and adding some more from scratch.
  */
-static const uint16_t quantization_per_subband[10][9] = {
-    { 12, 12,  6, 12, 12,  6,  12,  12,  18, },
-    { 14, 14,  7, 14, 14,  7,  18,  18,  24, },
-    { 16, 16,  8, 16, 16,  8,  24,  24,  36, },
-    { 16, 16,  8, 16, 16,  8,  48,  48,  96, },
-    { 24, 24, 16, 24, 24, 16,  64,  64, 128, },
-    { 24, 24, 16, 24, 24, 16, 128, 128, 256, },
-    { 64, 64, 48, 64, 64, 48, 256, 256, 512, },
-    { 64, 64, 48, 64, 64, 48, 512, 512, 768, },
-    { 64, 64, 48, 64, 64, 48,1024,1024,1536, },
-    {160,160,168,172,172,180,2048,2048,3072, },
+static const uint16_t quantization_per_subband[2][3][11][9] = {
+    {{
+        { 16, 16,  8,  4,  4,  2,   6,   6,   9, }, // film3+
+        { 16, 16,  8,  4,  4,  2,   6,   6,   9, }, // film3
+        { 16, 16,  8,  4,  4,  2,   8,   8,  12, }, // film2+
+        { 16, 16,  8,  4,  4,  2,   8,   8,  12, }, // film2
+        { 24, 24, 12,  6,  6,  3,  24,  24,  36, }, // film1+
+        { 24, 24, 12,  6,  6,  3,  24,  24,  36, }, // film1
+        { 32, 32, 24,  8,  8,  6,  32,  32,  48, }, // high+
+        { 32, 32, 24,  8,  8,  6,  32,  32,  48, }, // high
+        { 48, 48, 32, 12, 12,  8,  64,  64,  96, }, // medium+
+        { 48, 48, 32, 12, 12,  8,  64,  64,  96, }, // medium
+        { 64, 64, 48, 16, 16, 12, 128, 128, 192, }, // low
+    },
+    {
+        { 16, 16,  8,  4,  4,  2,   6,   6,   9, },
+        { 16, 16,  8,  4,  4,  2,   6,   6,  12, },
+        { 16, 16,  8,  4,  4,  2,   8,   8,  12, },
+        { 16, 16,  8,  4,  4,  2,   8,   8,  16, },
+        { 24, 24, 12,  6,  6,  3,  24,  24,  36, },
+        { 24, 24, 12,  6,  6,  3,  24,  24,  48, },
+        { 32, 32, 24,  8,  8,  6,  32,  32,  48, },
+        { 48, 48, 32, 12, 12,  8,  32,  32,  64, },
+        { 48, 48, 32, 12, 12,  8,  64,  64,  96, },
+        { 48, 48, 32, 12, 12,  8,  64,  64, 128, },
+        { 64, 64, 48, 16, 16, 12, 128, 128, 192, },
+    },
+    {
+        { 16, 16,  8,  4,  4,  2,   6,   6,   9, },
+        { 16, 16,  8,  4,  4,  2,   6,   6,  12, },
+        { 16, 16,  8,  4,  4,  2,   8,   8,  12, },
+        { 16, 16,  8,  4,  4,  2,   8,   8,  16, },
+        { 24, 24, 12,  6,  6,  3,  24,  24,  36, },
+        { 24, 24, 12,  6,  6,  3,  24,  24,  48, },
+        { 32, 32, 24,  8,  8,  6,  32,  32,  48, },
+        { 48, 48, 32, 12, 12,  8,  32,  32,  64, },
+        { 48, 48, 32, 12, 12,  8,  64,  64,  96, },
+        { 48, 48, 32, 12, 12,  8,  64,  64, 128, },
+        { 64, 64, 48, 16, 16, 12, 128, 128, 192, },
+    }},
+    {{
+        { 16, 16,  8, 16, 16,  8,  24,  24,  36, },
+        { 16, 16,  8, 16, 16,  8,  32,  32,  48, },
+        { 16, 16,  8, 16, 16,  8,  48,  48,  72, },
+        { 16, 16,  8, 16, 16,  8,  64,  64,  96, },
+        { 24, 24, 12, 24, 24, 12,  96,  96, 144, },
+        { 24, 24, 12, 24, 24, 12, 192, 192, 288, },
+        { 32, 32, 24, 32, 32, 24, 128, 128, 192, },
+        { 32, 32, 24, 32, 32, 24, 256, 256, 384, },
+        { 48, 48, 32, 48, 48, 32, 256, 256, 384, },
+        { 48, 48, 32, 48, 48, 32, 512, 512, 768, },
+        { 64, 64, 48, 64, 64, 48, 512, 512, 768, },
+    },
+    {
+        { 16, 16,  8, 16, 16,  8,  24,  24,  36, },
+        { 16, 16,  8, 16, 16,  8,  32,  32,  48, },
+        { 16, 16,  8, 16, 16,  8,  48,  48,  72, },
+        { 16, 16,  8, 16, 16,  8,  64,  64,  96, },
+        { 24, 24, 12, 24, 24, 12,  96,  96, 144, },
+        { 24, 24, 12, 24, 24, 12, 192, 192, 288, },
+        { 32, 32, 24, 32, 32, 24, 128, 128, 192, },
+        { 32, 32, 24, 32, 32, 24, 256, 256, 384, },
+        { 48, 48, 32, 48, 48, 32, 256, 256, 384, },
+        { 48, 48, 32, 48, 48, 32, 512, 512, 768, },
+        { 64, 64, 48, 64, 64, 48, 512, 512, 768, },
+    },
+    {
+        { 16, 16,  8, 16, 16,  8,  24,  24,  36, },
+        { 16, 16,  8, 16, 16,  8,  32,  32,  48, },
+        { 16, 16,  8, 16, 16,  8,  48,  48,  72, },
+        { 16, 16,  8, 16, 16,  8,  64,  64,  96, },
+        { 24, 24, 12, 24, 24, 12,  96,  96, 144, },
+        { 24, 24, 12, 24, 24, 12, 192, 192, 288, },
+        { 32, 32, 24, 32, 32, 24, 128, 128, 192, },
+        { 32, 32, 24, 32, 32, 24, 256, 256, 384, },
+        { 48, 48, 32, 48, 48, 32, 256, 256, 384, },
+        { 48, 48, 32, 48, 48, 32, 512, 512, 768, },
+        { 64, 64, 48, 64, 64, 48, 512, 512, 768, },
+    }},
 };
 
 typedef struct Codebook {
@@ -144,7 +212,6 @@ typedef struct CFHDEncContext {
     PutBitContext       pb;
     PutByteContext      pby;
 
-    unsigned quantization_div;
     int compression;
     int planes;
     int chroma_h_shift;
@@ -163,8 +230,6 @@ static av_cold int cfhd_encode_init(AVCodecContext *avctx)
     const int mag_mask = sign_mask - 1;
     int ret;
 
-    s->quantization_div = avctx->pix_fmt == AV_PIX_FMT_YUV422P10 ? 2 : 1;
-
     ret = av_pix_fmt_get_chroma_sub_sample(avctx->pix_fmt,
                                            &s->chroma_h_shift,
                                            &s->chroma_v_shift);
@@ -180,7 +245,7 @@ static av_cold int cfhd_encode_init(AVCodecContext *avctx)
 
     if (avctx->compression_level == FF_COMPRESSION_DEFAULT)
         s->compression = 2;
-    s->compression = av_clip(avctx->compression_level, 0, 9);
+    s->compression = av_clip(avctx->compression_level, 0, 10);
 
     for (int i = 0; i < s->planes; i++) {
         int w8, h8, w4, h4, w2, h2;
@@ -584,7 +649,7 @@ static int cfhd_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
 
         for (int l = 0; l < 3; l++) {
             for (int i = 0; i < 3; i++) {
-                s->plane[p].quantization[1 + l * 3 + i] = quantization_per_subband[s->compression][l * 3 + i] / s->quantization_div;
+                s->plane[p].quantization[1 + l * 3 + i] = quantization_per_subband[avctx->pix_fmt != AV_PIX_FMT_YUV422P10][p][s->compression][l * 3 + i];
             }
         }