lavfi/framesync: map mode WIP

2 files changed, 162 insertions, 7 deletions

diff --git a/libavfilter/framesync.c b/libavfilter/framesync.c
index fdcc3b57c8..8d29926000 100644
--- a/libavfilter/framesync.c
+++ b/libavfilter/framesync.c
@@ -18,7 +18,11 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#define _POSIX_C_SOURCE 200809L
+#include <stdio.h>
+
 #include "libavutil/avassert.h"
+#include "libavutil/file_open.h"
 #include "libavutil/opt.h"
 #include "avfilter.h"
 #include "filters.h"
@@ -49,6 +53,7 @@ static const AVOption framesync_options[] = {
             0, AV_OPT_TYPE_CONST, { .i64 = TS_DEFAULT }, .flags = FLAGS, "ts_sync_mode" },
         { "nearest", "Frame from secondary input with the absolute nearest timestamp to the primary input frame",
             0, AV_OPT_TYPE_CONST, { .i64 = TS_NEAREST }, .flags = FLAGS, "ts_sync_mode" },
+    { "frame_map_path", "", OFFSET(frame_map_path), AV_OPT_TYPE_STRING, .flags = FLAGS },
     { NULL }
 };
 static const AVClass framesync_class = {
@@ -129,10 +134,107 @@ static void framesync_sync_level_update(FFFrameSync *fs)
         framesync_eof(fs);
 }
 
+#define WHITESPACE " \t\r\n"
+
+static int frame_map_read(FFFrameSync *fs, const char *map_path)
+{
+    char  *line       = NULL;
+    size_t line_alloc = 0;
+    int ret = 0;
+
+    FILE *map_file;
+    ssize_t len;
+
+    map_file = avpriv_fopen_utf8(map_path, "rb");
+    if (!map_file) {
+        av_log(fs, AV_LOG_ERROR, "Error opening the frame map file\n");
+        return AVERROR(ENOENT);
+    }
+
+    while ((len = getline(&line, &line_alloc, map_file)) != -1) {
+        const char *p = line;
+        size_t skip;
+        uint64_t *dst;
+
+        skip = strspn(p, WHITESPACE);
+        p   += skip;
+        len -= skip;
+
+        // skip comments and empty lines
+        if (len <= 0 || p[0] == '#')
+            continue;
+
+        dst = av_fast_realloc(fs->frame_map, &fs->frame_map_allocated,
+                              sizeof(*fs->frame_map) * fs->nb_in * (fs->nb_frame_map + 1));
+        if (!dst) {
+            ret = AVERROR(ENOMEM);
+            goto finish;
+        }
+        fs->frame_map = dst;
+        dst += fs->nb_in * fs->nb_frame_map;
+        fs->nb_frame_map++;
+
+        // read a frame number for each input
+        for (int i = 0; i < fs->nb_in; i++) {
+            char *p1;
+            dst[i] = strtoul(p, &p1, 0);
+            if (p1 == p) {
+                av_log(fs, AV_LOG_ERROR,
+                       "Invalid number in frame map on line %zu: %s\n",
+                       fs->nb_frame_map - 1, p);
+                ret = AVERROR_INVALIDDATA;
+                goto finish;
+            }
+            p = p1;
+
+            if (fs->nb_frame_map > 1 && dst[i - (int)fs->nb_in] > dst[i]) {
+                av_log(fs, AV_LOG_ERROR,
+                       "Frame map for input %d, frame %zu goes backwards\n",
+                       i, fs->nb_frame_map - 1);
+                ret = AVERROR_INVALIDDATA;
+                goto finish;
+            }
+
+            skip = strspn(p, WHITESPACE);
+            p   += skip;
+            len -= skip;
+        }
+
+        if (*p) {
+            av_log(fs, AV_LOG_ERROR, "Trailing garbage on line %s: %s\n",
+                   line, p);
+            ret = AVERROR_INVALIDDATA;
+            goto finish;
+        }
+    }
+
+finish:
+    fclose(map_file);
+    free(line);
+
+    return ret;
+}
+
 int ff_framesync_configure(FFFrameSync *fs)
 {
     unsigned i;
 
+    if (fs->frame_map_path) {
+        int ret;
+
+        if (fs->opt_ts_sync_mode != TS_DEFAULT) {
+            av_log(fs, AV_LOG_ERROR,
+                   "ts_sync_mode must be set to default when a map is used\n");
+            return AVERROR(EINVAL);
+        }
+
+        ret = frame_map_read(fs, fs->frame_map_path);
+        if (ret < 0) {
+            av_log(fs, AV_LOG_ERROR, "Error reading the explicit frame map\n");
+            return ret;
+        }
+    }
+
     if (!fs->opt_repeatlast || fs->opt_eof_action == EOF_ACTION_PASS) {
         fs->opt_repeatlast = 0;
         fs->opt_eof_action = EOF_ACTION_PASS;
@@ -171,8 +273,10 @@ int ff_framesync_configure(FFFrameSync *fs)
                fs->time_base.num, fs->time_base.den);
     }
 
-    for (i = 0; i < fs->nb_in; i++)
+    for (i = 0; i < fs->nb_in; i++) {
         fs->in[i].pts = fs->in[i].pts_next = AV_NOPTS_VALUE;
+        fs->in[i].frame_number = UINT64_MAX;
+    }
     fs->sync_level = UINT_MAX;
     framesync_sync_level_update(fs);
 
@@ -250,17 +354,54 @@ static int consume_from_fifos(FFFrameSync *fs)
     return 1;
 }
 
+static void frame_advance(FFFrameSyncIn *in)
+{
+    av_frame_free(&in->frame);
+    in->frame      = in->frame_next;
+    in->pts        = in->pts_next;
+    in->frame_next = NULL;
+    in->pts_next   = AV_NOPTS_VALUE;
+    in->have_next  = 0;
+
+    if (in->frame) {
+        in->frame_number = in->frame_number == UINT64_MAX ?
+                           0 : in->frame_number + 1;
+    }
+}
+
 static int framesync_advance(FFFrameSync *fs)
 {
     unsigned i;
     int64_t pts;
     int ret;
 
+    if (fs->frame_map && fs->nb_events >= fs->nb_frame_map) {
+        framesync_eof(fs);
+        return 0;
+    }
+
     while (!(fs->frame_ready || fs->eof)) {
         ret = consume_from_fifos(fs);
         if (ret <= 0)
             return ret;
 
+        if (fs->frame_map) {
+            fs->frame_ready = 1;
+            for (i = 0; i < fs->nb_in; i++) {
+                FFFrameSyncIn * const in = &fs->in[i];
+                uint64_t next_number = fs->frame_map[fs->nb_events * fs->nb_in + i];
+                if (in->frame_number != next_number) {
+                    frame_advance(in);
+                    fs->frame_ready = 0;
+                    in->state       = in->frame ? STATE_RUN : STATE_EOF;
+                    if (in->state == STATE_EOF) {
+                        av_log(fs, AV_LOG_WARNING, "Input stream %d ended before the map did\n", i);
+                        framesync_eof(fs);
+                    }
+                }
+            }
+            pts = fs->in[0].pts;
+        } else {
         pts = INT64_MAX;
         for (i = 0; i < fs->nb_in; i++)
             if (fs->in[i].have_next && fs->in[i].pts_next < pts)
@@ -277,12 +418,7 @@ static int framesync_advance(FFFrameSync *fs)
                  in->pts_next != INT64_MAX && in->pts != AV_NOPTS_VALUE &&
                  in->pts_next - pts < pts - in->pts) ||
                 (in->before == EXT_INFINITY && in->state == STATE_BOF)) {
-                av_frame_free(&in->frame);
-                in->frame      = in->frame_next;
-                in->pts        = in->pts_next;
-                in->frame_next = NULL;
-                in->pts_next   = AV_NOPTS_VALUE;
-                in->have_next  = 0;
+                frame_advance(in);
                 in->state      = in->frame ? STATE_RUN : STATE_EOF;
                 if (in->sync == fs->sync_level && in->frame)
                     fs->frame_ready = 1;
@@ -295,6 +431,7 @@ static int framesync_advance(FFFrameSync *fs)
                 if ((fs->in[i].state == STATE_BOF &&
                      fs->in[i].before == EXT_STOP))
                     fs->frame_ready = 0;
+        }
         fs->pts = pts;
     }
     return 0;
@@ -347,6 +484,10 @@ void ff_framesync_uninit(FFFrameSync *fs)
     }
 
     av_freep(&fs->in);
+
+    av_freep(&fs->frame_map_path);
+    av_freep(&fs->frame_map);
+    fs->frame_map_allocated = 0;
 }
 
 int ff_framesync_activate(FFFrameSync *fs)
@@ -359,6 +500,7 @@ int ff_framesync_activate(FFFrameSync *fs)
     if (fs->eof || !fs->frame_ready)
         return 0;
     ret = fs->on_event(fs);
+    fs->nb_events++;
     if (ret < 0)
         return ret;
     fs->frame_ready = 0;
diff --git a/libavfilter/framesync.h b/libavfilter/framesync.h
index 233f50a0eb..2932da01c0 100644
--- a/libavfilter/framesync.h
+++ b/libavfilter/framesync.h
@@ -136,6 +136,8 @@ typedef struct FFFrameSyncIn {
      */
     int64_t pts_next;
 
+    uint64_t frame_number;
+
     /**
      * Boolean flagging the next frame, for internal use
      */
@@ -189,6 +191,11 @@ typedef struct FFFrameSync {
     int64_t pts;
 
     /**
+     * Number of times on_event() was called.
+     */
+    uint64_t nb_events;
+
+    /**
      * Callback called when a frame event is ready
      */
     int (*on_event)(struct FFFrameSync *fs);
@@ -229,6 +236,12 @@ typedef struct FFFrameSync {
     int opt_eof_action;
     int opt_ts_sync_mode;
 
+    char *frame_map_path;
+
+    // explicit frame map
+    uint64_t       *frame_map;
+    size_t       nb_frame_map;
+    unsigned int    frame_map_allocated;
 } FFFrameSync;
 
 /**