avformat/mov: rewrite aspect from matrix code

The original code was intended purely for rotation == 0
In cf70ba37ba74089a18295b29e77dead0a3222c9e the condition was
changed to use it only for rotation != 0
which broke the cases for which it was intended to be used
as well as breaking cases for which it was not intended to be
used.
This changes the code so it could work for the more general
case and fixes the regressions
If you have sample files that are not handled correctly
please open tickets or mail me!

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 10 insertions, 13 deletions

diff --git a/libavformat/mov.c b/libavformat/mov.c
index 8312d2b105..3da296bd07 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -2784,7 +2784,6 @@ static int mov_read_tkhd(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     int i;
     int width;
     int height;
-    int64_t disp_transform[2];
     int display_matrix[3][3];
     AVStream *st;
     MOVStreamContext *sc;
@@ -2867,20 +2866,18 @@ static int mov_read_tkhd(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     }
 
     // transform the display width/height according to the matrix
-    // skip this if the rotation angle is 0 degrees
     // to keep the same scale, use [width height 1<<16]
-    if (width && height && sc->display_matrix &&
-        av_display_rotation_get(sc->display_matrix) != 0.0f) {
+    if (width && height && sc->display_matrix) {
+        double disp_transform[2];
+
+#define SQR(a) ((a)*(double)(a))
         for (i = 0; i < 2; i++)
-            disp_transform[i] =
-                (int64_t)  width  * display_matrix[0][i] +
-                (int64_t)  height * display_matrix[1][i] +
-                ((int64_t) display_matrix[2][i] << 16);
-
-        //sample aspect ratio is new width/height divided by old width/height
-        st->sample_aspect_ratio = av_d2q(
-            ((double) disp_transform[0] * height) /
-            ((double) disp_transform[1] * width), INT_MAX);
+            disp_transform[i] = sqrt(SQR(display_matrix[i][0]) + SQR(display_matrix[i][1]));
+
+        if (fabs((disp_transform[0] / disp_transform[1]) - 1.0) > 0.01)
+            st->sample_aspect_ratio = av_d2q(
+                disp_transform[0] / disp_transform[1],
+                INT_MAX);
     }
     return 0;
 }