aacdec: convert to lavu/tx and support fixed-point 960-sample decoding

This patch replaces the transform used in AAC with lavu/tx and removes
the limitation on only being able to decode 960-sample files
with the float decoder.
This commit also removes a whole bunch of unnecessary and slow
lifting steps the decoder did to compensate for the poor accuracy
of the old integer transformation code.

Overall float decoder speedup on Zen 3 for 64kbps: 32%

2 files changed, 145 insertions, 145 deletions

diff --git a/tests/ref/fate/filter-meta-4560-rotate0 b/tests/ref/fate/filter-meta-4560-rotate0
index 82703e009c..c0f3b03710 100644
--- a/tests/ref/fate/filter-meta-4560-rotate0
+++ b/tests/ref/fate/filter-meta-4560-rotate0
@@ -16,254 +16,254 @@ dc213aee944a55af2f41950921fd62d7 *tests/data/fate/filter-meta-4560-rotate0.mov
 0,          1,          1,        1,   195840, 0xcad55625
 1,       2048,       2048,     1024,     2048, 0x9275068a
 0,          2,          2,        1,   195840, 0x1a47a5ca
-1,       3072,       3072,     1024,     2048, 0xcd35a7ce
-1,       4096,       4096,     1024,     2048, 0x3f846a1e
+1,       3072,       3072,     1024,     2048, 0xcecba7cf
+1,       4096,       4096,     1024,     2048, 0x39006a1d
 0,          3,          3,        1,   195840, 0x5cd02d7c
-1,       5120,       5120,     1024,     2048, 0x60ff5fd1
+1,       5120,       5120,     1024,     2048, 0x59795fd0
 0,          4,          4,        1,   195840, 0x07d08f16
-1,       6144,       6144,     1024,     2048, 0x9e9d96ee
-1,       7168,       7168,     1024,     2048, 0xb2622c29
+1,       6144,       6144,     1024,     2048, 0x9a7796ec
+1,       7168,       7168,     1024,     2048, 0xb7da2c2b
 0,          5,          5,        1,   195840, 0xe8abf6dd
-1,       8192,       8192,     1024,     2048, 0xdeddcd6a
+1,       8192,       8192,     1024,     2048, 0xe4efcd6a
 0,          6,          6,        1,   195840, 0x096a5c58
-1,       9216,       9216,     1024,     2048, 0xc8798b23
-1,      10240,      10240,     1024,     2048, 0xea1dac49
+1,       9216,       9216,     1024,     2048, 0xbfa78b21
+1,      10240,      10240,     1024,     2048, 0xeeddac49
 0,          7,          7,        1,   195840, 0x4e1a17bd
-1,      11264,      11264,     1024,     2048, 0x6b50813e
+1,      11264,      11264,     1024,     2048, 0x72d28140
 0,          8,          8,        1,   195840, 0x96349c20
-1,      12288,      12288,     1024,     2048, 0x89698347
+1,      12288,      12288,     1024,     2048, 0xa03f834c
 0,          9,          9,        1,   195840, 0x6bb16907
-1,      13312,      13312,     1024,     2048, 0xaa608a47
-1,      14336,      14336,     1024,     2048, 0x169b8cf7
+1,      13312,      13312,     1024,     2048, 0xb1168a48
+1,      14336,      14336,     1024,     2048, 0x1b258cf9
 0,         10,         10,        1,   195840, 0x7e44e4dd
-1,      15360,      15360,     1024,     2048, 0x2f127455
+1,      15360,      15360,     1024,     2048, 0x35e07458
 0,         11,         11,        1,   195840, 0x4c3c0cf1
-1,      16384,      16384,     1024,     2048, 0x269c7583
-1,      17408,      17408,     1024,     2048, 0x917f7279
+1,      16384,      16384,     1024,     2048, 0x21d07582
+1,      17408,      17408,     1024,     2048, 0x847f7276
 0,         12,         12,        1,   195840, 0x4f2e3f2d
-1,      18432,      18432,     1024,     2048, 0x27055ffb
+1,      18432,      18432,     1024,     2048, 0x080f5ff4
 0,         13,         13,        1,   195840, 0xcba539cd
-1,      19456,      19456,     1024,     2048, 0x7eca49e4
-1,      20480,      20480,     1024,     2048, 0xad067184
+1,      19456,      19456,     1024,     2048, 0x714049e3
+1,      20480,      20480,     1024,     2048, 0xaf8c7186
 0,         14,         14,        1,   195840, 0xf2811c9b
-1,      21504,      21504,     1024,     2048, 0xca406454
+1,      21504,      21504,     1024,     2048, 0xc3666452
 0,         15,         15,        1,   195840, 0x8357eab2
-1,      22528,      22528,     1024,     2048, 0x69d48a55
+1,      22528,      22528,     1024,     2048, 0x5b088a52
 0,         16,         16,        1,   195840, 0xdba9c438
-1,      23552,      23552,     1024,     2048, 0x0a836c14
-1,      24576,      24576,     1024,     2048, 0xa9e967f9
+1,      23552,      23552,     1024,     2048, 0x0bab6c14
+1,      24576,      24576,     1024,     2048, 0xaaa967f8
 0,         17,         17,        1,   195840, 0x58bcb594
-1,      25600,      25600,     1024,     2048, 0xd7f73f35
+1,      25600,      25600,     1024,     2048, 0xcc3d3f34
 0,         18,         18,        1,   195840, 0x3edc5a1c
-1,      26624,      26624,     1024,     2048, 0x70ed8364
-1,      27648,      27648,     1024,     2048, 0x913f9b73
+1,      26624,      26624,     1024,     2048, 0x6bdd8363
+1,      27648,      27648,     1024,     2048, 0xdc6a9d71
 0,         19,         19,        1,   195840, 0x94e4c186
-1,      28672,      28672,     1024,     2048, 0x7db083eb
+1,      28672,      28672,     1024,     2048, 0x751883ea
 0,         20,         20,        1,   195840, 0xffad5f59
-1,      29696,      29696,     1024,     2048, 0x951b97a9
-1,      30720,      30720,     1024,     2048, 0x072ea78e
+1,      29696,      29696,     1024,     2048, 0x922997a8
+1,      30720,      30720,     1024,     2048, 0x0626a78f
 0,         21,         21,        1,   195840, 0x84d1b114
-1,      31744,      31744,     1024,     2048, 0x9b127ba5
+1,      31744,      31744,     1024,     2048, 0x9c347ba6
 0,         22,         22,        1,   195840, 0x358c1d0a
-1,      32768,      32768,     1024,     2048, 0x051c90c0
-1,      33792,      33792,     1024,     2048, 0x232d93b5
+1,      32768,      32768,     1024,     2048, 0x020e90bf
+1,      33792,      33792,     1024,     2048, 0x2c4993b6
 0,         23,         23,        1,   195840, 0x3ed1ffc8
-1,      34816,      34816,     1024,     2048, 0x76b89477
+1,      34816,      34816,     1024,     2048, 0x7a989478
 0,         24,         24,        1,   195840, 0xf048f47c
-1,      35840,      35840,     1024,     2048, 0x50b2568d
+1,      35840,      35840,     1024,     2048, 0x3e7c568a
 0,         25,         25,        1,   195840, 0x3c2a3de6
-1,      36864,      36864,     1024,     2048, 0x97947e5e
-1,      37888,      37888,     1024,     2048, 0x21569819
+1,      36864,      36864,     1024,     2048, 0x8e5e7e5d
+1,      37888,      37888,     1024,     2048, 0x16349816
 0,         26,         26,        1,   195840, 0x7ca208b1
-1,      38912,      38912,     1024,     2048, 0xc4358de4
+1,      38912,      38912,     1024,     2048, 0xccef8de5
 0,         27,         27,        1,   195840, 0x105cf9fc
-1,      39936,      39936,     1024,     2048, 0x69964977
-1,      40960,      40960,     1024,     2048, 0xef81892e
+1,      39936,      39936,     1024,     2048, 0x6e904977
+1,      40960,      40960,     1024,     2048, 0xa69f872d
 0,         28,         28,        1,   195840, 0x9fe50c29
-1,      41984,      41984,     1024,     2048, 0x6f7084e7
+1,      41984,      41984,     1024,     2048, 0x7dc788e5
 0,         29,         29,        1,   195840, 0xb459720f
-1,      43008,      43008,     1024,     2048, 0x7be080f7
-1,      44032,      44032,     1024,     2048, 0xadb18939
+1,      43008,      43008,     1024,     2048, 0x7b7c80f7
+1,      44032,      44032,     1024,     2048, 0xa9ef8939
 0,         30,         30,        1,   195840, 0x6eda17b0
-1,      45056,      45056,     1024,     2048, 0x6cd9506d
+1,      45056,      45056,     1024,     2048, 0x6b9b506d
 0,         31,         31,        1,   195840, 0x098871b0
-1,      46080,      46080,     1024,     2048, 0xba88ab8b
+1,      46080,      46080,     1024,     2048, 0xb55cab8a
 0,         32,         32,        1,   195840, 0xc664cc79
-1,      47104,      47104,     1024,     2048, 0xe0bd726b
-1,      48128,      48128,     1024,     2048, 0x15604a65
+1,      47104,      47104,     1024,     2048, 0xd1eb7268
+1,      48128,      48128,     1024,     2048, 0x104e4a65
 0,         33,         33,        1,   195840, 0x8d02708d
-1,      49152,      49152,     1024,     2048, 0x2a89f85b
+1,      49152,      49152,     1024,     2048, 0x28f1f85a
 0,         34,         34,        1,   195840, 0xe90a8705
-1,      50176,      50176,     1024,     2048, 0xd71857dd
-1,      51200,      51200,     1024,     2048, 0x93d98aee
+1,      50176,      50176,     1024,     2048, 0x841a55de
+1,      51200,      51200,     1024,     2048, 0x90f58aed
 0,         35,         35,        1,   195840, 0x55f6c5b6
-1,      52224,      52224,     1024,     2048, 0xcc537c08
+1,      52224,      52224,     1024,     2048, 0xbeb17c04
 0,         36,         36,        1,   195840, 0xe4ad145d
-1,      53248,      53248,     1024,     2048, 0xe0ea7c21
-1,      54272,      54272,     1024,     2048, 0x883ba380
+1,      53248,      53248,     1024,     2048, 0xe9247c23
+1,      54272,      54272,     1024,     2048, 0x8995a383
 0,         37,         37,        1,   195840, 0x9e766d50
-1,      55296,      55296,     1024,     2048, 0xaaf65ca7
+1,      55296,      55296,     1024,     2048, 0xa4805ca7
 0,         38,         38,        1,   195840, 0xc2eac289
-1,      56320,      56320,     1024,     2048, 0xe01e98ca
+1,      56320,      56320,     1024,     2048, 0xedec98ca
 0,         39,         39,        1,   195840, 0x0baf5871
-1,      57344,      57344,     1024,     2048, 0xca85a551
-1,      58368,      58368,     1024,     2048, 0x6fea7f82
+1,      57344,      57344,     1024,     2048, 0xc495a551
+1,      58368,      58368,     1024,     2048, 0x76d07f84
 0,         40,         40,        1,   195840, 0x15f85f38
-1,      59392,      59392,     1024,     2048, 0x9fd59843
+1,      59392,      59392,     1024,     2048, 0x9a159842
 0,         41,         41,        1,   195840, 0x77b03072
-1,      60416,      60416,     1024,     2048, 0xd7419d35
-1,      61440,      61440,     1024,     2048, 0xc285bc0e
+1,      60416,      60416,     1024,     2048, 0xddcb9d36
+1,      61440,      61440,     1024,     2048, 0xc1abbc0d
 0,         42,         42,        1,   195840, 0x0abb7abf
-1,      62464,      62464,     1024,     2048, 0x5180419c
+1,      62464,      62464,     1024,     2048, 0x5754419e
 0,         43,         43,        1,   195840, 0x05076724
-1,      63488,      63488,     1024,     2048, 0xac9fd91b
+1,      63488,      63488,     1024,     2048, 0xabd3d91a
 1,      64512,      64512,     1024,     2048, 0x6de689b3
 0,         44,         44,        1,   195840, 0x2e8c6163
-1,      65536,      65536,     1024,     2048, 0x655bb415
+1,      65536,      65536,     1024,     2048, 0x6c67b415
 0,         45,         45,        1,   195840, 0x4f350cde
-1,      66560,      66560,     1024,     2048, 0xe59d7404
-1,      67584,      67584,     1024,     2048, 0xaa9356b9
+1,      66560,      66560,     1024,     2048, 0xdb637402
+1,      67584,      67584,     1024,     2048, 0xb56156bb
 0,         46,         46,        1,   195840, 0xefc6bda1
-1,      68608,      68608,     1024,     2048, 0xe193549d
+1,      68608,      68608,     1024,     2048, 0xebf5549e
 0,         47,         47,        1,   195840, 0xe2cdee68
-1,      69632,      69632,     1024,     2048, 0xd05c9736
+1,      69632,      69632,     1024,     2048, 0xccda9735
 0,         48,         48,        1,   195840, 0xcc62401e
-1,      70656,      70656,     1024,     2048, 0x086f6829
-1,      71680,      71680,     1024,     2048, 0x79624f93
+1,      70656,      70656,     1024,     2048, 0xff4e6828
+1,      71680,      71680,     1024,     2048, 0x77d84f92
 0,         49,         49,        1,   195840, 0x4ed76e1b
-1,      72704,      72704,     1024,     2048, 0xf70a2f21
+1,      72704,      72704,     1024,     2048, 0xf28a2f20
 0,         50,         50,        1,   195840, 0xf4f8599a
-1,      73728,      73728,     1024,     2048, 0xd32c643c
+1,      73728,      73728,     1024,     2048, 0xda70643c
 1,      74752,      74752,     1024,     2048, 0xd6f07035
 0,         51,         51,        1,   195840, 0xb05edf96
-1,      75776,      75776,     1024,     2048, 0x137c6ca0
+1,      75776,      75776,     1024,     2048, 0x23e66ca2
 0,         52,         52,        1,   195840, 0xfb1a1e15
-1,      76800,      76800,     1024,     2048, 0x916242de
-1,      77824,      77824,     1024,     2048, 0x0d910367
+1,      76800,      76800,     1024,     2048, 0x8f5842dd
+1,      77824,      77824,     1024,     2048, 0xfd580364
 0,         53,         53,        1,   195840, 0x970ebb58
-1,      78848,      78848,     1024,     2048, 0x460b8f1e
+1,      78848,      78848,     1024,     2048, 0x441d8f1e
 0,         54,         54,        1,   195840, 0xf4046957
-1,      79872,      79872,     1024,     2048, 0x616a6102
+1,      79872,      79872,     1024,     2048, 0x5fcc6101
 0,         55,         55,        1,   195840, 0xd5eca339
-1,      80896,      80896,     1024,     2048, 0xfc507e1d
-1,      81920,      81920,     1024,     2048, 0x34accc94
+1,      80896,      80896,     1024,     2048, 0x05097c1f
+1,      81920,      81920,     1024,     2048, 0x3226cc96
 0,         56,         56,        1,   195840, 0x52d3cb24
-1,      82944,      82944,     1024,     2048, 0xcf679f1a
+1,      82944,      82944,     1024,     2048, 0xa981a118
 0,         57,         57,        1,   195840, 0x259cac47
-1,      83968,      83968,     1024,     2048, 0x72c45d8a
-1,      84992,      84992,     1024,     2048, 0xfd21264a
+1,      83968,      83968,     1024,     2048, 0x7d705d8e
+1,      84992,      84992,     1024,     2048, 0xfd8d264a
 0,         58,         58,        1,   195840, 0x9190a95b
-1,      86016,      86016,     1024,     2048, 0x02895c31
+1,      86016,      86016,     1024,     2048, 0x01875c2f
 0,         59,         59,        1,   195840, 0x5baccfb5
-1,      87040,      87040,     1024,     2048, 0xdc041e9c
+1,      87040,      87040,     1024,     2048, 0xdaf01e9d
 1,      88064,      88064,     1024,     2048, 0x266407a5
 0,         60,         60,        1,   195840, 0x80d19a09
-1,      89088,      89088,     1024,     2048, 0x8f111c49
+1,      89088,      89088,     1024,     2048, 0x92951c49
 0,         61,         61,        1,   195840, 0x5ad44ce3
-1,      90112,      90112,     1024,     2048, 0xead031dd
-1,      91136,      91136,     1024,     2048, 0x238325d4
+1,      90112,      90112,     1024,     2048, 0xec2031dd
+1,      91136,      91136,     1024,     2048, 0x18bf25d2
 0,         62,         62,        1,   195840, 0x7b1ad12b
-1,      92160,      92160,     1024,     2048, 0x24171913
+1,      92160,      92160,     1024,     2048, 0x21a71912
 0,         63,         63,        1,   195840, 0x57e1a5e5
-1,      93184,      93184,     1024,     2048, 0xbcda41c5
+1,      93184,      93184,     1024,     2048, 0xc25841c7
 0,         64,         64,        1,   195840, 0x70cb6d38
-1,      94208,      94208,     1024,     2048, 0x124f5eed
-1,      95232,      95232,     1024,     2048, 0x4cadc088
+1,      94208,      94208,     1024,     2048, 0x05955eeb
+1,      95232,      95232,     1024,     2048, 0x490dc087
 0,         65,         65,        1,   195840, 0x39dac35a
-1,      96256,      96256,     1024,     2048, 0x062cd95a
+1,      96256,      96256,     1024,     2048, 0x055ed95a
 0,         66,         66,        1,   195840, 0x4f9fccca
-1,      97280,      97280,     1024,     2048, 0xc26a888d
-1,      98304,      98304,     1024,     2048, 0x864f1a68
+1,      97280,      97280,     1024,     2048, 0xc356888f
+1,      98304,      98304,     1024,     2048, 0x93bf1a6c
 0,         67,         67,        1,   195840, 0xadcb5585
-1,      99328,      99328,     1024,     2048, 0x4bb1ecf4
+1,      99328,      99328,     1024,     2048, 0x4a5decf6
 0,         68,         68,        1,   195840, 0xe596c1b4
-1,     100352,     100352,     1024,     2048, 0xb4ddc0a7
-1,     101376,     101376,     1024,     2048, 0x665badb3
+1,     100352,     100352,     1024,     2048, 0xfcf1c2a5
+1,     101376,     101376,     1024,     2048, 0x5ba9adaf
 0,         69,         69,        1,   195840, 0x64a05339
-1,     102400,     102400,     1024,     2048, 0x70beaa68
+1,     102400,     102400,     1024,     2048, 0x4da1a86b
 0,         70,         70,        1,   195840, 0xaefa0990
-1,     103424,     103424,     1024,     2048, 0x8ac7c7de
+1,     103424,     103424,     1024,     2048, 0x7bcfc7da
 0,         71,         71,        1,   195840, 0x9c0b5c3d
-1,     104448,     104448,     1024,     2048, 0x158cad8e
-1,     105472,     105472,     1024,     2048, 0x0ab8faf2
+1,     104448,     104448,     1024,     2048, 0x2274ad91
+1,     105472,     105472,     1024,     2048, 0x0484faf2
 0,         72,         72,        1,   195840, 0xbae8bd6b
-1,     106496,     106496,     1024,     2048, 0x994e79f1
+1,     106496,     106496,     1024,     2048, 0x994479f0
 0,         73,         73,        1,   195840, 0xb2ac7857
-1,     107520,     107520,     1024,     2048, 0xc727104e
-1,     108544,     108544,     1024,     2048, 0x1e1f720a
+1,     107520,     107520,     1024,     2048, 0xcfff104f
+1,     108544,     108544,     1024,     2048, 0x17877208
 0,         74,         74,        1,   195840, 0xcf2865e3
-1,     109568,     109568,     1024,     2048, 0x7232bede
+1,     109568,     109568,     1024,     2048, 0x79eabedf
 0,         75,         75,        1,   195840, 0x4e8589af
-1,     110592,     110592,     1024,     2048, 0x015fd697
-1,     111616,     111616,     1024,     2048, 0x4b33ebc9
+1,     110592,     110592,     1024,     2048, 0xfe84d697
+1,     111616,     111616,     1024,     2048, 0x5751ebcc
 0,         76,         76,        1,   195840, 0x31c21938
-1,     112640,     112640,     1024,     2048, 0x8e61948a
+1,     112640,     112640,     1024,     2048, 0x8ae59488
 0,         77,         77,        1,   195840, 0xe6163c50
-1,     113664,     113664,     1024,     2048, 0x83fab6ac
+1,     113664,     113664,     1024,     2048, 0x8870b6ae
 0,         78,         78,        1,   195840, 0x888ef74f
-1,     114688,     114688,     1024,     2048, 0x8d55d683
+1,     114688,     114688,     1024,     2048, 0x9255d684
 1,     115712,     115712,     1024,     2048, 0x4e91a2c7
 0,         79,         79,        1,   195840, 0x00d822f7
-1,     116736,     116736,     1024,     2048, 0x8c79c692
+1,     116736,     116736,     1024,     2048, 0x9895c693
 0,         80,         80,        1,   195840, 0xcbe77d2b
-1,     117760,     117760,     1024,     2048, 0xe744d102
-1,     118784,     118784,     1024,     2048, 0x2bf4e6b1
+1,     117760,     117760,     1024,     2048, 0xee96d102
+1,     118784,     118784,     1024,     2048, 0x3508e6b3
 0,         81,         81,        1,   195840, 0x9f2ff6e5
-1,     119808,     119808,     1024,     2048, 0xd6fd9b91
+1,     119808,     119808,     1024,     2048, 0xd7a79b90
 0,         82,         82,        1,   195840, 0x08f65f5f
-1,     120832,     120832,     1024,     2048, 0xedcd8e8a
-1,     121856,     121856,     1024,     2048, 0x67b422bd
+1,     120832,     120832,     1024,     2048, 0xe77f8e88
+1,     121856,     121856,     1024,     2048, 0x623c22bd
 0,         83,         83,        1,   195840, 0xd3950469
-1,     122880,     122880,     1024,     2048, 0xa0329606
+1,     122880,     122880,     1024,     2048, 0x9f989605
 0,         84,         84,        1,   195840, 0xb8e014b8
-1,     123904,     123904,     1024,     2048, 0xae47c015
+1,     123904,     123904,     1024,     2048, 0xbbe1c017
 1,     124928,     124928,     1024,     2048, 0xaec1dfb2
 0,         85,         85,        1,   195840, 0xa4399742
-1,     125952,     125952,     1024,     2048, 0x719e9bac
+1,     125952,     125952,     1024,     2048, 0x7c709bae
 0,         86,         86,        1,   195840, 0x84b5a8ec
-1,     126976,     126976,     1024,     2048, 0x5d99a16c
+1,     126976,     126976,     1024,     2048, 0x55a1a16b
 0,         87,         87,        1,   195840, 0x1bfaa027
-1,     128000,     128000,     1024,     2048, 0x87aacd09
-1,     129024,     129024,     1024,     2048, 0x1473be08
+1,     128000,     128000,     1024,     2048, 0x9a9ccd0e
+1,     129024,     129024,     1024,     2048, 0x1087be07
 0,         88,         88,        1,   195840, 0x7bdbb172
-1,     130048,     130048,     1024,     2048, 0xdee3d975
+1,     130048,     130048,     1024,     2048, 0xcd3bd974
 0,         89,         89,        1,   195840, 0x00f2b3f7
-1,     131072,     131072,     1024,     2048, 0x0180b050
-1,     132096,     132096,     1024,     2048, 0xb035d915
+1,     131072,     131072,     1024,     2048, 0xfd43b04e
+1,     132096,     132096,     1024,     2048, 0xb2add916
 0,         90,         90,        1,   195840, 0x86503bc7
-1,     133120,     133120,     1024,     2048, 0x1b77a3fd
+1,     133120,     133120,     1024,     2048, 0x156da3fc
 0,         91,         91,        1,   195840, 0xc4ab358e
-1,     134144,     134144,     1024,     2048, 0xf233b7bc
-1,     135168,     135168,     1024,     2048, 0xf1fad173
+1,     134144,     134144,     1024,     2048, 0xf29fb7bb
+1,     135168,     135168,     1024,     2048, 0xf19ed172
 0,         92,         92,        1,   195840, 0x1b330ba0
-1,     136192,     136192,     1024,     2048, 0x8b6ac5b9
+1,     136192,     136192,     1024,     2048, 0x8094c5b7
 0,         93,         93,        1,   195840, 0xccc9822f
-1,     137216,     137216,     1024,     2048, 0x1228c03f
+1,     137216,     137216,     1024,     2048, 0x16d0c040
 0,         94,         94,        1,   195840, 0xffe3486d
-1,     138240,     138240,     1024,     2048, 0xbe99cc61
-1,     139264,     139264,     1024,     2048, 0x8486bad7
+1,     138240,     138240,     1024,     2048, 0xc895cc63
+1,     139264,     139264,     1024,     2048, 0xd260b8d9
 0,         95,         95,        1,   195840, 0x30dc0efe
-1,     140288,     140288,     1024,     2048, 0x423ca644
+1,     140288,     140288,     1024,     2048, 0x3dcca644
 0,         96,         96,        1,   195840, 0x6bfae71e
-1,     141312,     141312,     1024,     2048, 0x33afb5a3
-1,     142336,     142336,     1024,     2048, 0x90c33f49
+1,     141312,     141312,     1024,     2048, 0x241bb5a1
+1,     142336,     142336,     1024,     2048, 0x980b3f4a
 0,         97,         97,        1,   195840, 0x0fa017b8
-1,     143360,     143360,     1024,     2048, 0x0e59f9d7
+1,     143360,     143360,     1024,     2048, 0x05edf9d5
 0,         98,         98,        1,   195840, 0x1d0c9810
-1,     144384,     144384,     1024,     2048, 0x4f513d3d
-1,     145408,     145408,     1024,     2048, 0xe2f1406b
+1,     144384,     144384,     1024,     2048, 0x4f8b3d3d
+1,     145408,     145408,     1024,     2048, 0xe6b5406c
 0,         99,         99,        1,   195840, 0x775778f0
-1,     146432,     146432,     1024,     2048, 0xe55d2a99
+1,     146432,     146432,     1024,     2048, 0xe9792a9a
 0,        100,        100,        1,   195840, 0xdcb0ede5
-1,     147456,     147456,     1024,     2048, 0xb0e236c2
+1,     147456,     147456,     1024,     2048, 0xb54e36c3
 0,        101,        101,        1,   195840, 0x67624270
-1,     148480,     148480,     1024,     2048, 0xdc80c8fb
-1,     149504,     149504,     1024,     2048, 0x97c30778
+1,     148480,     148480,     1024,     2048, 0xd8ecc8f8
+1,     149504,     149504,     1024,     2048, 0x9f150779
 0,        102,        102,        1,   195840, 0xce86f6a3
 1,     150528,     150528,     1024,     2048, 0x6e403113
 0,        103,        103,        1,   195840, 0xa4b3e032
-1,     151552,     151552,     1024,     2048, 0x429605a1
-1,     152576,     152576,     1024,     2048, 0x25d61b75
+1,     151552,     151552,     1024,     2048, 0x473a05a3
+1,     152576,     152576,     1024,     2048, 0x26721b75
 0,        104,        104,        1,   195840, 0x46851b87
-1,     153600,     153600,     1024,     2048, 0xa32cf42d
+1,     153600,     153600,     1024,     2048, 0x97a2f42c
 0,        105,        105,        1,   195840, 0x02d97dc4
diff --git a/tests/ref/fate/prores-gray b/tests/ref/fate/prores-gray
index 08fcf2473d..ae7becc47a 100644
--- a/tests/ref/fate/prores-gray
+++ b/tests/ref/fate/prores-gray
@@ -9,7 +9,7 @@
 #sample_rate 1: 48000
 #channel_layout_name 1: stereo
 0,          0,          0,        1,  1658880, 0x43d9c9e2
-1,          0,          0,     1024,     4096, 0x6c8a9a18
-1,       1024,       1024,     1024,     4096, 0x960dadcf
+1,          0,          0,     1024,     4096, 0x543c9a17
+1,       1024,       1024,     1024,     4096, 0x9aedadd1
 0,          1,          1,        1,  1658880, 0xa2a72f9b
-1,       2048,       2048,     1024,     4096, 0xa913cd55
+1,       2048,       2048,     1024,     4096, 0xa0c9cd55