j2k/jpeg2000: split data pointer in int & float.

This fixes a TODO item and unifies both decoders structures
It also fixes undefined behavior due to aliasing violations

I choose 2 fields instead of a union because mistakely using the
wrong type with a union will lead to hard to debug "wrong output"
while with 2 fields mistakely using the wrong type will crash
with a null pointer derefernce which is much easier to debug

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

7 files changed, 59 insertions, 42 deletions

diff --git a/libavcodec/j2k.c b/libavcodec/j2k.c
index d5be7c3394..4c9c62b9a4 100644
--- a/libavcodec/j2k.c
+++ b/libavcodec/j2k.c
@@ -189,9 +189,17 @@ int ff_j2k_init_component(Jpeg2000Component *comp,
     for (i = 0; i < 2; i++)
         csize *= comp->coord[i][1] - comp->coord[i][0];
 
-    comp->data = av_malloc_array(csize, sizeof(*comp->data));
-    if (!comp->data)
-        return AVERROR(ENOMEM);
+    if (codsty->transform == FF_DWT97) {
+        comp->i_data = NULL;
+        comp->f_data = av_malloc_array(csize, sizeof(*comp->f_data));
+        if (!comp->f_data)
+            return AVERROR(ENOMEM);
+    } else {
+        comp->f_data = NULL;
+        comp->i_data = av_malloc_array(csize, sizeof(*comp->i_data));
+        if (!comp->i_data)
+            return AVERROR(ENOMEM);
+    }
     comp->reslevel = av_malloc_array(codsty->nreslevels, sizeof(*comp->reslevel));
     if (!comp->reslevel)
         return AVERROR(ENOMEM);
@@ -484,5 +492,6 @@ void ff_j2k_cleanup(Jpeg2000Component *comp, Jpeg2000CodingStyle *codsty)
 
     ff_dwt_destroy(&comp->dwt);
     av_freep(&comp->reslevel);
-    av_freep(&comp->data);
+    av_freep(&comp->i_data);
+    av_freep(&comp->f_data);
 }
diff --git a/libavcodec/j2k.h b/libavcodec/j2k.h
index 65ecc7ab8b..d5a09be6d1 100644
--- a/libavcodec/j2k.h
+++ b/libavcodec/j2k.h
@@ -186,7 +186,7 @@ typedef struct Jpeg2000Band {
 
 typedef struct Jpeg2000ResLevel {
     uint8_t nbands;
-    uint16_t coord[2][2];   // border coordinates {{x0, x1}, {y0, y1}}
+    uint16_t coord[2][2]; // border coordinates {{x0, x1}, {y0, y1}}
     uint16_t num_precincts_x, num_precincts_y; // number of precincts in x/y direction
     uint8_t log2_prec_width, log2_prec_height; // exponent of precinct size
     Jpeg2000Band *band;
@@ -195,7 +195,8 @@ typedef struct Jpeg2000ResLevel {
 typedef struct Jpeg2000Component {
     Jpeg2000ResLevel *reslevel;
     DWTContext dwt;
-    int *data;
+    float *f_data;
+    int *i_data;
     uint16_t coord[2][2];   // border coordinates {{x0, x1}, {y0, y1}} -- can be reduced with lowres option
     uint16_t coord_o[2][2]; // border coordinates {{x0, x1}, {y0, y1}} -- original values from jpeg2000 headers
 } Jpeg2000Component;
diff --git a/libavcodec/j2kdec.c b/libavcodec/j2kdec.c
index 2f2b581783..5b7097952d 100644
--- a/libavcodec/j2kdec.c
+++ b/libavcodec/j2kdec.c
@@ -825,7 +825,7 @@ static void dequantization_float(int x, int y, Jpeg2000Cblk *cblk,
                                  Jpeg2000T1Context *t1, Jpeg2000Band *band)
 {
     int i, j, idx;
-    float *datap = &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
+    float *datap = &comp->f_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
     for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j)
         for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
             idx        = (comp->coord[0][1] - comp->coord[0][0]) * j + i;
@@ -839,8 +839,7 @@ static void dequantization_int(int x, int y, Jpeg2000Cblk *cblk,
                                Jpeg2000T1Context *t1, Jpeg2000Band *band)
 {
     int i, j, idx;
-    int32_t *datap =
-        (int32_t *) &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
+    int32_t *datap = &comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
     for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j)
         for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
             idx        = (comp->coord[0][1] - comp->coord[0][0]) * j + i;
@@ -872,9 +871,9 @@ static void mct_decode(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
 
     for (i = 0; i < 3; i++)
         if (tile->codsty[0].transform == FF_DWT97)
-            srcf[i] = tile->comp[i].data;
+            srcf[i] = tile->comp[i].f_data;
         else
-            src[i] = (int32_t *)tile->comp[i].data;
+            src [i] = tile->comp[i].i_data;
 
     for (i = 0; i < 2; i++)
         csize *= tile->comp[0].coord[i][1] - tile->comp[0].coord[i][0];
@@ -967,7 +966,7 @@ static int decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
             } /* end band */
         } /* end reslevel */
 
-        ff_dwt_decode(&comp->dwt, comp->data);
+        ff_dwt_decode(&comp->dwt, codsty->transform == FF_DWT97 ? (void*)comp->f_data : (void*)comp->i_data);
     } /*end comp */
 
     /* inverse MCT transformation */
@@ -977,8 +976,8 @@ static int decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
     if (s->precision <= 8) {
         for (compno = 0; compno < s->ncomponents; compno++) {
             Jpeg2000Component *comp = tile->comp + compno;
-            float *datap = (float*)comp->data;
-            int32_t *i_datap = (int32_t *) comp->data;
+            float *datap = comp->f_data;
+            int32_t *i_datap = comp->i_data;
 
             y = tile->comp[compno].coord[1][0] - s->image_offset_y;
             line = s->picture->data[0] + y * s->picture->linesize[0];
@@ -1007,8 +1006,8 @@ static int decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
     } else {
         for (compno = 0; compno < s->ncomponents; compno++) {
             Jpeg2000Component *comp = tile->comp + compno;
-            float *datap = (float*)comp->data;
-            int32_t *i_datap = (int32_t *) comp->data;
+            float *datap = comp->f_data;
+            int32_t *i_datap = comp->i_data;
             uint16_t *linel;
 
             y     = tile->comp[compno].coord[1][0] - s->image_offset_y;
diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c
index d31b84a121..38cc7c2529 100644
--- a/libavcodec/j2kenc.c
+++ b/libavcodec/j2kenc.c
@@ -98,7 +98,7 @@ static void printcomp(Jpeg2000Component *comp)
 {
     int i;
     for (i = 0; i < comp->y1 - comp->y0; i++)
-        ff_j2k_printv(comp->data + i * (comp->x1 - comp->x0), comp->x1 - comp->x0);
+        ff_j2k_printv(comp->i_data + i * (comp->x1 - comp->x0), comp->x1 - comp->x0);
 }
 
 static void dump(Jpeg2000EncoderContext *s, FILE *fd)
@@ -389,7 +389,7 @@ static void copy_frame(Jpeg2000EncoderContext *s)
         if (s->planar){
             for (compno = 0; compno < s->ncomponents; compno++){
                 Jpeg2000Component *comp = tile->comp + compno;
-                int *dst = comp->data;
+                int *dst = comp->i_data;
                 line = s->picture.data[compno]
                        + comp->coord[1][0] * s->picture.linesize[compno]
                        + comp->coord[0][0];
@@ -409,7 +409,7 @@ static void copy_frame(Jpeg2000EncoderContext *s)
                 uint8_t *ptr = line;
                 for (x = tile->comp[0].coord[0][0]; x < tile->comp[0].coord[0][1]; x++, i++){
                     for (compno = 0; compno < s->ncomponents; compno++){
-                        tile->comp[compno].data[i] = *ptr++  - (1 << 7);
+                        tile->comp[compno].i_data[i] = *ptr++  - (1 << 7);
                     }
                 }
                 line += s->picture.linesize[0];
@@ -819,7 +819,7 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno
         Jpeg2000Component *comp = s->tile[tileno].comp + compno;
 
         av_log(s->avctx, AV_LOG_DEBUG,"dwt\n");
-        if (ret = ff_dwt_encode(&comp->dwt, comp->data))
+        if (ret = ff_dwt_encode(&comp->dwt, comp->i_data))
             return ret;
         av_log(s->avctx, AV_LOG_DEBUG,"after dwt -> tier1\n");
 
@@ -855,14 +855,14 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno
                             for (y = yy0; y < yy1; y++){
                                 int *ptr = t1.data[y-yy0];
                                 for (x = xx0; x < xx1; x++){
-                                    *ptr++ = comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x] << NMSEDEC_FRACBITS;
+                                    *ptr++ = comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x] << NMSEDEC_FRACBITS;
                                 }
                             }
                         } else{
                             for (y = yy0; y < yy1; y++){
                                 int *ptr = t1.data[y-yy0];
                                 for (x = xx0; x < xx1; x++){
-                                    *ptr = (comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x]);
+                                    *ptr = (comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x]);
                                     *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->i_stepsize) >> 14 - NMSEDEC_FRACBITS;
                                     ptr++;
                                 }
diff --git a/libavcodec/jpeg2000.c b/libavcodec/jpeg2000.c
index bcbb25c806..62b6874a96 100644
--- a/libavcodec/jpeg2000.c
+++ b/libavcodec/jpeg2000.c
@@ -200,9 +200,17 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
     csize = (comp->coord[0][1] - comp->coord[0][0]) *
             (comp->coord[1][1] - comp->coord[1][0]);
 
-    comp->data = av_malloc_array(csize, sizeof(*comp->data));
-    if (!comp->data)
-        return AVERROR(ENOMEM);
+    if (codsty->transform == FF_DWT97) {
+        comp->i_data = NULL;
+        comp->f_data = av_malloc_array(csize, sizeof(*comp->f_data));
+        if (!comp->f_data)
+            return AVERROR(ENOMEM);
+    } else {
+        comp->f_data = NULL;
+        comp->i_data = av_malloc_array(csize, sizeof(*comp->i_data));
+        if (!comp->i_data)
+            return AVERROR(ENOMEM);
+    }
     comp->reslevel = av_malloc_array(codsty->nreslevels, sizeof(*comp->reslevel));
     if (!comp->reslevel)
         return AVERROR(ENOMEM);
@@ -470,5 +478,6 @@ void ff_jpeg2000_cleanup(Jpeg2000Component *comp, Jpeg2000CodingStyle *codsty)
 
     ff_dwt_destroy(&comp->dwt);
     av_freep(&comp->reslevel);
-    av_freep(&comp->data);
+    av_freep(&comp->i_data);
+    av_freep(&comp->f_data);
 }
diff --git a/libavcodec/jpeg2000.h b/libavcodec/jpeg2000.h
index 274bae96c5..ff313b59b5 100644
--- a/libavcodec/jpeg2000.h
+++ b/libavcodec/jpeg2000.h
@@ -192,13 +192,11 @@ typedef struct Jpeg2000ResLevel {
     Jpeg2000Band *band;
 } Jpeg2000ResLevel; // resolution level
 
-/* TODO: data can be float of integer depending of reversible/irreversible
- * transformation.
- */
 typedef struct Jpeg2000Component {
     Jpeg2000ResLevel *reslevel;
     DWTContext dwt;
-    float *data;
+    float *f_data;
+    int *i_data;
     uint16_t coord[2][2];   // border coordinates {{x0, x1}, {y0, y1}} -- can be reduced with lowres option
     uint16_t coord_o[2][2]; // border coordinates {{x0, x1}, {y0, y1}} -- original values from jpeg2000 headers
 } Jpeg2000Component;
diff --git a/libavcodec/jpeg2000dec.c b/libavcodec/jpeg2000dec.c
index 071524659b..4e8104c0c3 100644
--- a/libavcodec/jpeg2000dec.c
+++ b/libavcodec/jpeg2000dec.c
@@ -926,7 +926,7 @@ static void dequantization_float(int x, int y, Jpeg2000Cblk *cblk,
                                  Jpeg2000T1Context *t1, Jpeg2000Band *band)
 {
     int i, j, idx;
-    float *datap = &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
+    float *datap = &comp->f_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
     for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j)
         for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
             idx        = (comp->coord[0][1] - comp->coord[0][0]) * j + i;
@@ -940,8 +940,7 @@ static void dequantization_int(int x, int y, Jpeg2000Cblk *cblk,
                                Jpeg2000T1Context *t1, Jpeg2000Band *band)
 {
     int i, j, idx;
-    int32_t *datap =
-        (int32_t *) &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
+    int32_t *datap = &comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
     for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j)
         for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
             idx        = (comp->coord[0][1] - comp->coord[0][0]) * j + i;
@@ -973,9 +972,9 @@ static void mct_decode(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
 
     for (i = 0; i < 3; i++)
         if (tile->codsty[0].transform == FF_DWT97)
-            srcf[i] = tile->comp[i].data;
+            srcf[i] = tile->comp[i].f_data;
         else
-            src[i] = (int32_t *)tile->comp[i].data;
+            src [i] = tile->comp[i].i_data;
 
     for (i = 0; i < 2; i++)
         csize *= tile->comp[0].coord[i][1] - tile->comp[0].coord[i][0];
@@ -1066,21 +1065,23 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
         } /* end reslevel */
 
         /* inverse DWT */
-        ff_dwt_decode(&comp->dwt, comp->data);
+        ff_dwt_decode(&comp->dwt, codsty->transform == FF_DWT97 ? (void*)comp->f_data : (void*)comp->i_data);
     } /*end comp */
 
     /* inverse MCT transformation */
     if (tile->codsty[0].mct)
         mct_decode(s, tile);
 
-    if (s->avctx->pix_fmt == AV_PIX_FMT_BGRA) // RGBA -> BGRA
-        FFSWAP(float *, tile->comp[0].data, tile->comp[2].data);
+    if (s->avctx->pix_fmt == AV_PIX_FMT_BGRA) { // RGBA -> BGRA
+        FFSWAP(float *, tile->comp[0].f_data, tile->comp[2].f_data);
+        FFSWAP(int *, tile->comp[0].i_data, tile->comp[2].i_data);
+    }
 
     if (s->precision <= 8) {
         for (compno = 0; compno < s->ncomponents; compno++) {
             Jpeg2000Component *comp = tile->comp + compno;
-            float *datap = comp->data;
-            int32_t *i_datap = (int32_t *) comp->data;
+            float *datap = comp->f_data;
+            int32_t *i_datap = comp->i_data;
             y    = tile->comp[compno].coord[1][0] - s->image_offset_y;
             line = picture->data[0] + y * picture->linesize[0];
             for (; y < tile->comp[compno].coord[1][1] - s->image_offset_y; y += s->cdy[compno]) {
@@ -1108,8 +1109,8 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
     } else {
         for (compno = 0; compno < s->ncomponents; compno++) {
             Jpeg2000Component *comp = tile->comp + compno;
-            float *datap = comp->data;
-            int32_t *i_datap = (int32_t *) comp->data;
+            float *datap = comp->f_data;
+            int32_t *i_datap = comp->i_data;
             uint16_t *linel;
 
             y     = tile->comp[compno].coord[1][0] - s->image_offset_y;