From b5ac521a2311a09cbc4a3427d2d77bbd3a99cfd4 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Thu, 8 Dec 2022 18:13:22 +0100 Subject: lavfi/framesync: map mode WIP --- libavfilter/framesync.c | 156 +++++++++++++++++++++++++++++++++++++++++++++--- libavfilter/framesync.h | 13 ++++ 2 files changed, 162 insertions(+), 7 deletions(-) diff --git a/libavfilter/framesync.c b/libavfilter/framesync.c index fdcc3b57c8..8d29926000 100644 --- a/libavfilter/framesync.c +++ b/libavfilter/framesync.c @@ -18,7 +18,11 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#define _POSIX_C_SOURCE 200809L +#include + #include "libavutil/avassert.h" +#include "libavutil/file_open.h" #include "libavutil/opt.h" #include "avfilter.h" #include "filters.h" @@ -49,6 +53,7 @@ static const AVOption framesync_options[] = { 0, AV_OPT_TYPE_CONST, { .i64 = TS_DEFAULT }, .flags = FLAGS, "ts_sync_mode" }, { "nearest", "Frame from secondary input with the absolute nearest timestamp to the primary input frame", 0, AV_OPT_TYPE_CONST, { .i64 = TS_NEAREST }, .flags = FLAGS, "ts_sync_mode" }, + { "frame_map_path", "", OFFSET(frame_map_path), AV_OPT_TYPE_STRING, .flags = FLAGS }, { NULL } }; static const AVClass framesync_class = { @@ -129,10 +134,107 @@ static void framesync_sync_level_update(FFFrameSync *fs) framesync_eof(fs); } +#define WHITESPACE " \t\r\n" + +static int frame_map_read(FFFrameSync *fs, const char *map_path) +{ + char *line = NULL; + size_t line_alloc = 0; + int ret = 0; + + FILE *map_file; + ssize_t len; + + map_file = avpriv_fopen_utf8(map_path, "rb"); + if (!map_file) { + av_log(fs, AV_LOG_ERROR, "Error opening the frame map file\n"); + return AVERROR(ENOENT); + } + + while ((len = getline(&line, &line_alloc, map_file)) != -1) { + const char *p = line; + size_t skip; + uint64_t *dst; + + skip = strspn(p, WHITESPACE); + p += skip; + len -= skip; + + // skip comments and empty lines + if (len <= 0 || p[0] == '#') + continue; + + dst = av_fast_realloc(fs->frame_map, &fs->frame_map_allocated, + sizeof(*fs->frame_map) * fs->nb_in * (fs->nb_frame_map + 1)); + if (!dst) { + ret = AVERROR(ENOMEM); + goto finish; + } + fs->frame_map = dst; + dst += fs->nb_in * fs->nb_frame_map; + fs->nb_frame_map++; + + // read a frame number for each input + for (int i = 0; i < fs->nb_in; i++) { + char *p1; + dst[i] = strtoul(p, &p1, 0); + if (p1 == p) { + av_log(fs, AV_LOG_ERROR, + "Invalid number in frame map on line %zu: %s\n", + fs->nb_frame_map - 1, p); + ret = AVERROR_INVALIDDATA; + goto finish; + } + p = p1; + + if (fs->nb_frame_map > 1 && dst[i - (int)fs->nb_in] > dst[i]) { + av_log(fs, AV_LOG_ERROR, + "Frame map for input %d, frame %zu goes backwards\n", + i, fs->nb_frame_map - 1); + ret = AVERROR_INVALIDDATA; + goto finish; + } + + skip = strspn(p, WHITESPACE); + p += skip; + len -= skip; + } + + if (*p) { + av_log(fs, AV_LOG_ERROR, "Trailing garbage on line %s: %s\n", + line, p); + ret = AVERROR_INVALIDDATA; + goto finish; + } + } + +finish: + fclose(map_file); + free(line); + + return ret; +} + int ff_framesync_configure(FFFrameSync *fs) { unsigned i; + if (fs->frame_map_path) { + int ret; + + if (fs->opt_ts_sync_mode != TS_DEFAULT) { + av_log(fs, AV_LOG_ERROR, + "ts_sync_mode must be set to default when a map is used\n"); + return AVERROR(EINVAL); + } + + ret = frame_map_read(fs, fs->frame_map_path); + if (ret < 0) { + av_log(fs, AV_LOG_ERROR, "Error reading the explicit frame map\n"); + return ret; + } + } + if (!fs->opt_repeatlast || fs->opt_eof_action == EOF_ACTION_PASS) { fs->opt_repeatlast = 0; fs->opt_eof_action = EOF_ACTION_PASS; @@ -171,8 +273,10 @@ int ff_framesync_configure(FFFrameSync *fs) fs->time_base.num, fs->time_base.den); } - for (i = 0; i < fs->nb_in; i++) + for (i = 0; i < fs->nb_in; i++) { fs->in[i].pts = fs->in[i].pts_next = AV_NOPTS_VALUE; + fs->in[i].frame_number = UINT64_MAX; + } fs->sync_level = UINT_MAX; framesync_sync_level_update(fs); @@ -250,17 +354,54 @@ static int consume_from_fifos(FFFrameSync *fs) return 1; } +static void frame_advance(FFFrameSyncIn *in) +{ + av_frame_free(&in->frame); + in->frame = in->frame_next; + in->pts = in->pts_next; + in->frame_next = NULL; + in->pts_next = AV_NOPTS_VALUE; + in->have_next = 0; + + if (in->frame) { + in->frame_number = in->frame_number == UINT64_MAX ? + 0 : in->frame_number + 1; + } +} + static int framesync_advance(FFFrameSync *fs) { unsigned i; int64_t pts; int ret; + if (fs->frame_map && fs->nb_events >= fs->nb_frame_map) { + framesync_eof(fs); + return 0; + } + while (!(fs->frame_ready || fs->eof)) { ret = consume_from_fifos(fs); if (ret <= 0) return ret; + if (fs->frame_map) { + fs->frame_ready = 1; + for (i = 0; i < fs->nb_in; i++) { + FFFrameSyncIn * const in = &fs->in[i]; + uint64_t next_number = fs->frame_map[fs->nb_events * fs->nb_in + i]; + if (in->frame_number != next_number) { + frame_advance(in); + fs->frame_ready = 0; + in->state = in->frame ? STATE_RUN : STATE_EOF; + if (in->state == STATE_EOF) { + av_log(fs, AV_LOG_WARNING, "Input stream %d ended before the map did\n", i); + framesync_eof(fs); + } + } + } + pts = fs->in[0].pts; + } else { pts = INT64_MAX; for (i = 0; i < fs->nb_in; i++) if (fs->in[i].have_next && fs->in[i].pts_next < pts) @@ -277,12 +418,7 @@ static int framesync_advance(FFFrameSync *fs) in->pts_next != INT64_MAX && in->pts != AV_NOPTS_VALUE && in->pts_next - pts < pts - in->pts) || (in->before == EXT_INFINITY && in->state == STATE_BOF)) { - av_frame_free(&in->frame); - in->frame = in->frame_next; - in->pts = in->pts_next; - in->frame_next = NULL; - in->pts_next = AV_NOPTS_VALUE; - in->have_next = 0; + frame_advance(in); in->state = in->frame ? STATE_RUN : STATE_EOF; if (in->sync == fs->sync_level && in->frame) fs->frame_ready = 1; @@ -295,6 +431,7 @@ static int framesync_advance(FFFrameSync *fs) if ((fs->in[i].state == STATE_BOF && fs->in[i].before == EXT_STOP)) fs->frame_ready = 0; + } fs->pts = pts; } return 0; @@ -347,6 +484,10 @@ void ff_framesync_uninit(FFFrameSync *fs) } av_freep(&fs->in); + + av_freep(&fs->frame_map_path); + av_freep(&fs->frame_map); + fs->frame_map_allocated = 0; } int ff_framesync_activate(FFFrameSync *fs) @@ -359,6 +500,7 @@ int ff_framesync_activate(FFFrameSync *fs) if (fs->eof || !fs->frame_ready) return 0; ret = fs->on_event(fs); + fs->nb_events++; if (ret < 0) return ret; fs->frame_ready = 0; diff --git a/libavfilter/framesync.h b/libavfilter/framesync.h index 233f50a0eb..2932da01c0 100644 --- a/libavfilter/framesync.h +++ b/libavfilter/framesync.h @@ -136,6 +136,8 @@ typedef struct FFFrameSyncIn { */ int64_t pts_next; + uint64_t frame_number; + /** * Boolean flagging the next frame, for internal use */ @@ -188,6 +190,11 @@ typedef struct FFFrameSync { */ int64_t pts; + /** + * Number of times on_event() was called. + */ + uint64_t nb_events; + /** * Callback called when a frame event is ready */ @@ -229,6 +236,12 @@ typedef struct FFFrameSync { int opt_eof_action; int opt_ts_sync_mode; + char *frame_map_path; + + // explicit frame map + uint64_t *frame_map; + size_t nb_frame_map; + unsigned int frame_map_allocated; } FFFrameSync; /** -- cgit v1.2.3