From bee01ee2ba2e62974447d5e8ea2afb27dbdb1e23 Mon Sep 17 00:00:00 2001
From: Tobias Rapp <t.rapp@noa-archive.com>
Date: Tue, 29 Aug 2017 14:45:13 +0200
Subject: fate: add tests for psnr and ssim filter

Metadata filter output is passed through an Awk script comparing floats
against reference values with specified "fuzz" tolerance to account for
architectural differences (e.g. x86-32 vs. x86-64).

Signed-off-by: Tobias Rapp <t.rapp@noa-archive.com>
---
 tests/fate-run.sh                     |  9 +++++
 tests/fate/filter-video.mak           | 14 ++++++++
 tests/ref/fate/filter-refcmp-psnr-rgb | 45 ++++++++++++++++++++++++
 tests/ref/fate/filter-refcmp-psnr-yuv | 45 ++++++++++++++++++++++++
 tests/ref/fate/filter-refcmp-ssim-rgb | 30 ++++++++++++++++
 tests/ref/fate/filter-refcmp-ssim-yuv | 30 ++++++++++++++++
 tests/refcmp-metadata.awk             | 64 +++++++++++++++++++++++++++++++++++
 7 files changed, 237 insertions(+)
 create mode 100644 tests/ref/fate/filter-refcmp-psnr-rgb
 create mode 100644 tests/ref/fate/filter-refcmp-psnr-yuv
 create mode 100644 tests/ref/fate/filter-refcmp-ssim-rgb
 create mode 100644 tests/ref/fate/filter-refcmp-ssim-yuv
 create mode 100644 tests/refcmp-metadata.awk

diff --git a/tests/fate-run.sh b/tests/fate-run.sh
index 9aa9a22395..c5480c7ede 100755
--- a/tests/fate-run.sh
+++ b/tests/fate-run.sh
@@ -234,6 +234,15 @@ lavftest(){
     ${base}/lavf-regression.sh $t lavf tests/vsynth1 "$target_exec" "$target_path" "$threads" "$thread_type" "$cpuflags" "$target_samples"
 }
 
+refcmp_metadata(){
+    refcmp=$1
+    pixfmt=$2
+    fuzz=${3:-0.001}
+    ffmpeg $FLAGS $ENC_OPTS \
+        -lavfi "testsrc2=size=300x200:rate=1:duration=5,format=${pixfmt},split[ref][tmp];[tmp]avgblur=4[enc];[enc][ref]${refcmp},metadata=print:file=-" \
+        -f null /dev/null | awk -v ref=${ref} -v fuzz=${fuzz} -f ${base}/refcmp-metadata.awk -
+}
+
 video_filter(){
     filters=$1
     shift
diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak
index d1e13414f6..78cd4711e6 100644
--- a/tests/fate/filter-video.mak
+++ b/tests/fate/filter-video.mak
@@ -747,6 +747,20 @@ FATE_FILTER_SAMPLES-$(call ALLYES, MOV_DEMUXER H264_DECODER AAC_FIXED_DECODER PC
 fate-filter-meta-4560-rotate0: tests/data/file4560-override2rotate0.mov
 fate-filter-meta-4560-rotate0: CMD = framecrc -flags +bitexact -c:a aac_fixed -i $(TARGET_PATH)/tests/data/file4560-override2rotate0.mov
 
+REFCMP_DEPS = FFMPEG LAVFI_INDEV TESTSRC2_FILTER AVGBLUR_FILTER METADATA_FILTER
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-rgb
+fate-filter-refcmp-psnr-rgb: CMD = refcmp_metadata psnr rgb24
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-yuv
+fate-filter-refcmp-psnr-yuv: CMD = refcmp_metadata psnr yuv422p
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-rgb
+fate-filter-refcmp-ssim-rgb: CMD = refcmp_metadata ssim rgb24
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-yuv
+fate-filter-refcmp-ssim-yuv: CMD = refcmp_metadata ssim yuv422p
+
 FATE_SAMPLES_FFPROBE += $(FATE_METADATA_FILTER-yes)
 FATE_SAMPLES_FFMPEG += $(FATE_FILTER_SAMPLES-yes)
 FATE_FFMPEG += $(FATE_FILTER-yes)
diff --git a/tests/ref/fate/filter-refcmp-psnr-rgb b/tests/ref/fate/filter-refcmp-psnr-rgb
new file mode 100644
index 0000000000..f06db575ac
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-psnr-rgb
@@ -0,0 +1,45 @@
+frame:0    pts:0       pts_time:0
+lavfi.psnr.mse.r=1381.80
+lavfi.psnr.psnr.r=16.73
+lavfi.psnr.mse.g=896.00
+lavfi.psnr.psnr.g=18.61
+lavfi.psnr.mse.b=277.38
+lavfi.psnr.psnr.b=23.70
+lavfi.psnr.mse_avg=851.73
+lavfi.psnr.psnr_avg=18.83
+frame:1    pts:1       pts_time:1
+lavfi.psnr.mse.r=1380.37
+lavfi.psnr.psnr.r=16.73
+lavfi.psnr.mse.g=975.91
+lavfi.psnr.psnr.g=18.24
+lavfi.psnr.mse.b=435.72
+lavfi.psnr.psnr.b=21.74
+lavfi.psnr.mse_avg=930.67
+lavfi.psnr.psnr_avg=18.44
+frame:2    pts:2       pts_time:2
+lavfi.psnr.mse.r=1403.20
+lavfi.psnr.psnr.r=16.66
+lavfi.psnr.mse.g=954.05
+lavfi.psnr.psnr.g=18.34
+lavfi.psnr.mse.b=494.22
+lavfi.psnr.psnr.b=21.19
+lavfi.psnr.mse_avg=950.49
+lavfi.psnr.psnr_avg=18.35
+frame:3    pts:3       pts_time:3
+lavfi.psnr.mse.r=1452.80
+lavfi.psnr.psnr.r=16.51
+lavfi.psnr.mse.g=1001.02
+lavfi.psnr.psnr.g=18.13
+lavfi.psnr.mse.b=557.39
+lavfi.psnr.psnr.b=20.67
+lavfi.psnr.mse_avg=1003.74
+lavfi.psnr.psnr_avg=18.11
+frame:4    pts:4       pts_time:4
+lavfi.psnr.mse.r=1401.25
+lavfi.psnr.psnr.r=16.67
+lavfi.psnr.mse.g=1009.80
+lavfi.psnr.psnr.g=18.09
+lavfi.psnr.mse.b=602.42
+lavfi.psnr.psnr.b=20.33
+lavfi.psnr.mse_avg=1004.49
+lavfi.psnr.psnr_avg=18.11
diff --git a/tests/ref/fate/filter-refcmp-psnr-yuv b/tests/ref/fate/filter-refcmp-psnr-yuv
new file mode 100644
index 0000000000..0e634ed0e4
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-psnr-yuv
@@ -0,0 +1,45 @@
+frame:0    pts:0       pts_time:0
+lavfi.psnr.mse.y=222.06
+lavfi.psnr.psnr.y=24.67
+lavfi.psnr.mse.u=339.38
+lavfi.psnr.psnr.u=22.82
+lavfi.psnr.mse.v=705.41
+lavfi.psnr.psnr.v=19.65
+lavfi.psnr.mse_avg=372.23
+lavfi.psnr.psnr_avg=22.42
+frame:1    pts:1       pts_time:1
+lavfi.psnr.mse.y=236.74
+lavfi.psnr.psnr.y=24.39
+lavfi.psnr.mse.u=416.17
+lavfi.psnr.psnr.u=21.94
+lavfi.psnr.mse.v=704.98
+lavfi.psnr.psnr.v=19.65
+lavfi.psnr.mse_avg=398.66
+lavfi.psnr.psnr_avg=22.12
+frame:2    pts:2       pts_time:2
+lavfi.psnr.mse.y=234.79
+lavfi.psnr.psnr.y=24.42
+lavfi.psnr.mse.u=435.72
+lavfi.psnr.psnr.u=21.74
+lavfi.psnr.mse.v=699.60
+lavfi.psnr.psnr.v=19.68
+lavfi.psnr.mse_avg=401.23
+lavfi.psnr.psnr_avg=22.10
+frame:3    pts:3       pts_time:3
+lavfi.psnr.mse.y=250.88
+lavfi.psnr.psnr.y=24.14
+lavfi.psnr.mse.u=479.73
+lavfi.psnr.psnr.u=21.32
+lavfi.psnr.mse.v=707.55
+lavfi.psnr.psnr.v=19.63
+lavfi.psnr.mse_avg=422.26
+lavfi.psnr.psnr_avg=21.88
+frame:4    pts:4       pts_time:4
+lavfi.psnr.mse.y=241.05
+lavfi.psnr.psnr.y=24.31
+lavfi.psnr.mse.u=505.04
+lavfi.psnr.psnr.u=21.10
+lavfi.psnr.mse.v=716.00
+lavfi.psnr.psnr.v=19.58
+lavfi.psnr.mse_avg=425.79
+lavfi.psnr.psnr_avg=21.84
diff --git a/tests/ref/fate/filter-refcmp-ssim-rgb b/tests/ref/fate/filter-refcmp-ssim-rgb
new file mode 100644
index 0000000000..8c23c60b37
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-ssim-rgb
@@ -0,0 +1,30 @@
+frame:0    pts:0       pts_time:0
+lavfi.ssim.R=0.72
+lavfi.ssim.G=0.76
+lavfi.ssim.B=0.89
+lavfi.ssim.All=0.79
+lavfi.ssim.dB=6.74
+frame:1    pts:1       pts_time:1
+lavfi.ssim.R=0.70
+lavfi.ssim.G=0.74
+lavfi.ssim.B=0.85
+lavfi.ssim.All=0.77
+lavfi.ssim.dB=6.31
+frame:2    pts:2       pts_time:2
+lavfi.ssim.R=0.71
+lavfi.ssim.G=0.75
+lavfi.ssim.B=0.84
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.29
+frame:3    pts:3       pts_time:3
+lavfi.ssim.R=0.70
+lavfi.ssim.G=0.73
+lavfi.ssim.B=0.83
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.11
+frame:4    pts:4       pts_time:4
+lavfi.ssim.R=0.71
+lavfi.ssim.G=0.74
+lavfi.ssim.B=0.80
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.05
diff --git a/tests/ref/fate/filter-refcmp-ssim-yuv b/tests/ref/fate/filter-refcmp-ssim-yuv
new file mode 100644
index 0000000000..5c8ffb9483
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-ssim-yuv
@@ -0,0 +1,30 @@
+frame:0    pts:0       pts_time:0
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.76
+lavfi.ssim.V=0.69
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.25
+frame:1    pts:1       pts_time:1
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.73
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.08
+frame:2    pts:2       pts_time:2
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.73
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.10
+frame:3    pts:3       pts_time:3
+lavfi.ssim.Y=0.79
+lavfi.ssim.U=0.72
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=5.94
+frame:4    pts:4       pts_time:4
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.72
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=5.97
diff --git a/tests/refcmp-metadata.awk b/tests/refcmp-metadata.awk
new file mode 100644
index 0000000000..fa21aad0e0
--- /dev/null
+++ b/tests/refcmp-metadata.awk
@@ -0,0 +1,64 @@
+# Compare metadata filter output containing float value strings to reference
+# output data. Returns the whole reference data if delta of each value is below
+# threshold, else returns the whole input data.
+
+function abs(val) {
+    return ((val < 0.0) ? -val : val);
+}
+
+function max(val1, val2) {
+    return ((val1 >= val2) ? val1 : val2);
+}
+
+function is_numeric_str(str) {
+    return (str ~ /^[+-]?[0-9]*\.?[0-9]+$/);
+}
+
+BEGIN {
+    FS = "=";
+    # check for "fuzz" (threshold) program parameter, else use default
+    if (fuzz <= 0.0) {
+        fuzz = 0.1;
+    }
+    # check for "ref" (reference file) program parameter
+    if (ref) {
+        ref_nr = 0;
+        while ((getline line < ref) > 0) {
+            ref_nr++;
+            ref_lines[ref_nr] = line;
+            if (split(line, fields) == 2 && is_numeric_str(fields[2])) {
+                ref_keys[ref_nr] = fields[1];
+                ref_vals[ref_nr] = fields[2] + 0;  # convert to number
+            }
+        }
+        close(ref);
+    }
+    delta_max = 0;
+    result = (ref ? 1 : 0);
+}
+
+{
+    cmp_lines[NR] = $0;
+    if (NF == 2 && is_numeric_str($2) && ref_vals[NR]) {
+        val = $2 + 0;  # convert to number
+        delta = abs((val / ref_vals[NR]) - 1);
+        delta_max = max(delta_max, delta);
+        result = result && ($1 == ref_keys[NR]) && (delta <= fuzz);
+    } else {
+        result = result && ($0 == ref_lines[NR]);
+    }
+}
+
+END {
+    if (result) {
+        for (i = 1; i <= ref_nr; i++)
+            print ref_lines[i];
+    } else {
+        for (i = 1; i <= NR; i++)
+            print cmp_lines[i];
+        if (NR != ref_nr)
+            print "[refcmp] lines: " NR " != " ref_nr > "/dev/stderr";
+        if (delta_max >= fuzz)
+            print "[refcmp] delta_max: " delta_max " >= " fuzz > "/dev/stderr";
+    }
+}
-- 
cgit v1.2.3