From ab6f9d86a9db7d0743629ecab422cfb61f2a17e1 Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt Date: Sat, 18 Dec 2021 23:46:47 +0100 Subject: avcodec/h2645: Fix SEI->display matrix transformation The earlier code did not account for the fact that av_display_rotation_set() wants the angle in the anticlockwise direction (despite what its documentation stated for a long time); furthermore, the H.2645 spec wants the flips applied first, whereas our code did it the other way around. This can be fixed by negating the angle once for every flip. Signed-off-by: Andreas Rheinhardt --- libavcodec/hevcdec.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'libavcodec/hevcdec.c') diff --git a/libavcodec/hevcdec.c b/libavcodec/hevcdec.c index 46d9edf8eb..3aa70e2245 100644 --- a/libavcodec/hevcdec.c +++ b/libavcodec/hevcdec.c @@ -2769,6 +2769,16 @@ static int set_side_data(HEVCContext *s) if (!rotation) return AVERROR(ENOMEM); + /* av_display_rotation_set() expects the angle in the clockwise + * direction, hence the first minus. + * The below code applies the flips after the rotation, yet + * the H.2645 specs require flipping to be applied first. + * Because of R O(phi) = O(-phi) R (where R is flipping around + * an arbitatry axis and O(phi) is the proper rotation by phi) + * we can create display matrices as desired by negating + * the degree once for every flip applied. */ + angle = -angle * (1 - 2 * !!s->sei.display_orientation.hflip) + * (1 - 2 * !!s->sei.display_orientation.vflip); av_display_rotation_set((int32_t *)rotation->data, angle); av_display_matrix_flip((int32_t *)rotation->data, s->sei.display_orientation.hflip, -- cgit v1.2.3