lavfi/framesync: add syncing via external timestamp map

Useful when there is some external process that determines canonical
frame synchronization. E.g. the framerate conversion code in ffmpeg CLI.

3 files changed, 132 insertions, 6 deletions

diff --git a/doc/filters.texi b/doc/filters.texi
index 7a7b2ba4e7..afc6fd3ebf 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -374,6 +374,12 @@ primary input frame.
 Frame from secondary input with the absolute nearest timestamp to the primary
 input frame.
 @end table
+
+@item ts_map
+Specify an explicit timestamp map. The string should be composed of lines, one
+per each output frame. The line should contain whitespace-separated times in
+microseconds, one for every input. Frames with these timestamps will be matched
+together to produces output events.
 @end table
 
 @c man end OPTIONS FOR FILTERS WITH SEVERAL INPUTS
diff --git a/libavfilter/framesync.c b/libavfilter/framesync.c
index 08bbb2f54f..74c00f8e0a 100644
--- a/libavfilter/framesync.c
+++ b/libavfilter/framesync.c
@@ -49,6 +49,7 @@ static const AVOption framesync_options[] = {
             0, AV_OPT_TYPE_CONST, { .i64 = TS_DEFAULT }, .flags = FLAGS, "ts_sync_mode" },
         { "nearest", "Frame from secondary input with the absolute nearest timestamp to the primary input frame",
             0, AV_OPT_TYPE_CONST, { .i64 = TS_NEAREST }, .flags = FLAGS, "ts_sync_mode" },
+    { "ts_map", "Timestamp map", OFFSET(ts_map_str), AV_OPT_TYPE_STRING, .flags = FLAGS },
     { NULL }
 };
 static const AVClass framesync_class = {
@@ -129,10 +130,78 @@ static void framesync_sync_level_update(FFFrameSync *fs)
         framesync_eof(fs);
 }
 
+static int ts_map_parse(FFFrameSync *fs, const char *ts_map_str)
+{
+    while (*ts_map_str) {
+        int64_t *dst;
+
+        ts_map_str += strspn(ts_map_str, " \t\r\n");
+
+        // skip comments
+        if (*ts_map_str == '#' || !*ts_map_str)
+            goto skip_line;
+
+        dst = av_fast_realloc(fs->ts_map, &fs->ts_map_allocated,
+                              sizeof(*fs->ts_map) * fs->nb_in * (fs->nb_ts_map + 1));
+        if (!dst)
+            return AVERROR(ENOMEM);
+
+        fs->ts_map = dst;
+        dst += fs->nb_in * fs->nb_ts_map;
+        fs->nb_ts_map++;
+
+        // read a timestamp for each input
+        for (int i = 0; i < fs->nb_in; i++) {
+            char *p;
+            dst[i] = strtol(ts_map_str, &p, 0);
+            if (p == ts_map_str) {
+                av_log(fs, AV_LOG_ERROR,
+                       "Invalid number in timestamp map on line %zu: %s\n",
+                       fs->nb_ts_map - 1, ts_map_str);
+                return AVERROR_INVALIDDATA;
+            }
+            ts_map_str = p;
+
+            if (fs->nb_ts_map > 1 && dst[i - (int)fs->nb_in] > dst[i]) {
+                av_log(fs, AV_LOG_ERROR,
+                       "Timestamp map for input %d, frame %zu goes backwards\n",
+                       i, fs->nb_ts_map - 1);
+                return AVERROR_INVALIDDATA;
+            }
+
+            ts_map_str += strspn(p, " \t");
+        }
+
+        // skip everything after the needed timestamp
+skip_line:
+        ts_map_str = strchr(ts_map_str, '\n');
+        if (!ts_map_str)
+            break;
+    }
+
+    return 0;
+}
+
 int ff_framesync_configure(FFFrameSync *fs)
 {
     unsigned i;
 
+    if (fs->ts_map_str) {
+        int ret;
+
+        if (fs->opt_ts_sync_mode != TS_DEFAULT) {
+            av_log(fs, AV_LOG_ERROR,
+                   "ts_sync_mode must be set to default when a map is used\n");
+            return AVERROR(EINVAL);
+        }
+
+        ret = ts_map_parse(fs, fs->ts_map_str);
+        if (ret < 0) {
+            av_log(fs, AV_LOG_ERROR, "Error reading the explicit timestamp map\n");
+            return ret;
+        }
+    }
+
     if (!fs->opt_repeatlast || fs->opt_eof_action == EOF_ACTION_PASS) {
         fs->opt_repeatlast = 0;
         fs->opt_eof_action = EOF_ACTION_PASS;
@@ -250,17 +319,55 @@ static int consume_from_fifos(FFFrameSync *fs)
     return 1;
 }
 
+static void frame_advance(FFFrameSyncIn *in)
+{
+    av_frame_free(&in->frame);
+    in->frame      = in->frame_next;
+    in->pts        = in->pts_next;
+    in->frame_next = NULL;
+    in->pts_next   = AV_NOPTS_VALUE;
+    in->have_next  = 0;
+}
+
 static int framesync_advance(FFFrameSync *fs)
 {
     unsigned i;
     int64_t pts;
     int ret;
 
+    if (fs->ts_map && fs->nb_events >= fs->nb_ts_map) {
+        framesync_eof(fs);
+        return 0;
+    }
+
     while (!(fs->frame_ready || fs->eof)) {
         ret = consume_from_fifos(fs);
         if (ret <= 0)
             return ret;
 
+        if (fs->ts_map) {
+            fs->frame_ready = 1;
+            for (i = 0; i < fs->nb_in; i++) {
+                FFFrameSyncIn * const in = &fs->in[i];
+                int64_t next_ts = av_rescale_q(fs->ts_map[fs->nb_events * fs->nb_in + i],
+                                               AV_TIME_BASE_Q, fs->time_base);
+                uint64_t delta_cur  = in->frame      ? FFABS(in->pts      - next_ts) : UINT64_MAX;
+                uint64_t delta_next = in->frame_next ? FFABS(in->pts_next - next_ts) : UINT64_MAX;
+
+                if (!in->frame ||
+                    (in->frame_next && delta_next < delta_cur)) {
+                    frame_advance(in);
+                    fs->frame_ready = 0;
+                    in->state       = in->frame ? STATE_RUN : STATE_EOF;
+                    if (in->state == STATE_EOF) {
+                        av_log(fs, AV_LOG_WARNING,
+                               "Input stream %d ended before the timestamp map did\n", i);
+                        framesync_eof(fs);
+                    }
+                }
+            }
+            pts = fs->in[0].pts;
+        } else {
         pts = INT64_MAX;
         for (i = 0; i < fs->nb_in; i++)
             if (fs->in[i].have_next && fs->in[i].pts_next < pts)
@@ -277,12 +384,7 @@ static int framesync_advance(FFFrameSync *fs)
                  in->pts_next != INT64_MAX && in->pts != AV_NOPTS_VALUE &&
                  in->pts_next - pts < pts - in->pts) ||
                 (in->before == EXT_INFINITY && in->state == STATE_BOF)) {
-                av_frame_free(&in->frame);
-                in->frame      = in->frame_next;
-                in->pts        = in->pts_next;
-                in->frame_next = NULL;
-                in->pts_next   = AV_NOPTS_VALUE;
-                in->have_next  = 0;
+                frame_advance(in);
                 in->state      = in->frame ? STATE_RUN : STATE_EOF;
                 if (in->sync == fs->sync_level && in->frame)
                     fs->frame_ready = 1;
@@ -295,6 +397,7 @@ static int framesync_advance(FFFrameSync *fs)
                 if ((fs->in[i].state == STATE_BOF &&
                      fs->in[i].before == EXT_STOP))
                     fs->frame_ready = 0;
+        }
         fs->pts = pts;
     }
     return 0;
@@ -347,6 +450,11 @@ void ff_framesync_uninit(FFFrameSync *fs)
     }
 
     av_freep(&fs->in);
+
+    av_freep(&fs->ts_map_str);
+    av_freep(&fs->ts_map);
+    fs->nb_ts_map        = 0;
+    fs->ts_map_allocated = 0;
 }
 
 int ff_framesync_activate(FFFrameSync *fs)
@@ -359,6 +467,7 @@ int ff_framesync_activate(FFFrameSync *fs)
     if (fs->eof || !fs->frame_ready)
         return 0;
     ret = fs->on_event(fs);
+    fs->nb_events++;
     if (ret < 0)
         return ret;
     fs->frame_ready = 0;
diff --git a/libavfilter/framesync.h b/libavfilter/framesync.h
index 233f50a0eb..979f54e16e 100644
--- a/libavfilter/framesync.h
+++ b/libavfilter/framesync.h
@@ -189,6 +189,11 @@ typedef struct FFFrameSync {
     int64_t pts;
 
     /**
+     * Number of times on_event() was called.
+     */
+    uint64_t nb_events;
+
+    /**
      * Callback called when a frame event is ready
      */
     int (*on_event)(struct FFFrameSync *fs);
@@ -229,6 +234,12 @@ typedef struct FFFrameSync {
     int opt_eof_action;
     int opt_ts_sync_mode;
 
+    char *ts_map_str;
+
+    // explicit frame map
+    int64_t        *ts_map;
+    size_t       nb_ts_map;
+    unsigned int    ts_map_allocated;
 } FFFrameSync;
 
 /**