lavu/tx: implement 32 bit fixed point FFT and MDCT

Required minimal changes to the code so made sense to implement.
FFT and MDCT tested, the output of both was properly rounded.
Fun fact: the non-power-of-two fixed-point FFT and MDCT are the fastest ever
non-power-of-two fixed-point FFT and MDCT written.
This can replace the power of two integer MDCTs in aac and ac3 if the
MIPS optimizations are ported across.
Unfortunately the ac3 encoder uses a 16-bit fixed point forward transform,
unlike the encoder which uses a 32bit inverse transform, so some modifications
might be required there.

The 3-point FFT is somewhat less accurate than it otherwise could be,
having minor rounding errors with bigger transforms. However, this
could be improved later, and the way its currently written is the way one
would write assembly for it.
Similar rounding errors can also be found throughout the power of two FFTs
as well, though those are more difficult to correct.
Despite this, the integer transforms are more than accurate enough.

1 files changed, 18 insertions, 2 deletions

diff --git a/libavutil/tx.c b/libavutil/tx.c
index b8683b416b..3b0568a5e1 100644
--- a/libavutil/tx.c
+++ b/libavutil/tx.c
@@ -18,6 +18,18 @@
 
 #include "tx_priv.h"
 
+int ff_tx_type_is_mdct(enum AVTXType type)
+{
+    switch (type) {
+    case AV_TX_FLOAT_MDCT:
+    case AV_TX_DOUBLE_MDCT:
+    case AV_TX_INT32_MDCT:
+        return 1;
+    default:
+        return 0;
+    }
+}
+
 /* Calculates the modular multiplicative inverse, not fast, replace */
 static av_always_inline int mulinv(int n, int m)
 {
@@ -35,11 +47,10 @@ int ff_tx_gen_compound_mapping(AVTXContext *s)
     const int n     = s->n;
     const int m     = s->m;
     const int inv   = s->inv;
-    const int type  = s->type;
     const int len   = n*m;
     const int m_inv = mulinv(m, n);
     const int n_inv = mulinv(n, m);
-    const int mdct  = type == AV_TX_FLOAT_MDCT || type == AV_TX_DOUBLE_MDCT;
+    const int mdct  = ff_tx_type_is_mdct(s->type);
 
     if (!(s->pfatab = av_malloc(2*len*sizeof(*s->pfatab))))
         return AVERROR(ENOMEM);
@@ -128,6 +139,11 @@ av_cold int av_tx_init(AVTXContext **ctx, av_tx_fn *tx, enum AVTXType type,
         if ((err = ff_tx_init_mdct_fft_double(s, tx, type, inv, len, scale, flags)))
             goto fail;
         break;
+    case AV_TX_INT32_FFT:
+    case AV_TX_INT32_MDCT:
+        if ((err = ff_tx_init_mdct_fft_int32(s, tx, type, inv, len, scale, flags)))
+            goto fail;
+        break;
     default:
         err = AVERROR(EINVAL);
         goto fail;