From 43ea19fef4ce233510df99f68f85548e834a1995 Mon Sep 17 00:00:00 2001 From: Paul B Mahol Date: Sat, 9 Apr 2022 06:31:14 +0200 Subject: avfilter: add colormap video filter --- Changelog | 1 + doc/filters.texi | 33 +++ libavfilter/Makefile | 1 + libavfilter/allfilters.c | 1 + libavfilter/version.h | 2 +- libavfilter/vf_colormap.c | 582 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 619 insertions(+), 1 deletion(-) create mode 100644 libavfilter/vf_colormap.c diff --git a/Changelog b/Changelog index 7ce7052387..e028d7dfe0 100644 --- a/Changelog +++ b/Changelog @@ -11,6 +11,7 @@ version 5.1: - avsynctest source filter - feedback video filter - pixelize video filter +- colormap video filter version 5.0: diff --git a/doc/filters.texi b/doc/filters.texi index 79cabcfd6c..92d0e976d8 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -8944,6 +8944,39 @@ colorlevels=romin=0.5:gomin=0.5:bomin=0.5 This filter supports the all above options as @ref{commands}. +@section colormap +Apply custom color maps to video stream. + +This filter needs three input video streams. +First stream is video stream that is going to be filtered out. +Second and third video stream specify color patches for source +color to target color mapping. + +The filter accepts the following options: + +@table @option +@item patch_size +Set the source and target video stream patch size in pixels. + +@item nb_patches +Set the max number of used patches from source and target video stream. + +@item type +Set the adjustments used for target colors. Can be @code{relative} or @code{absolute}. +Defaults is @code{absolute}. + +@item kernel +Set the kernel used to measure color differences between mapped colors. + +The accepted values are: +@table @samp +@item euclidean +@item weuclidean +@end table + +Default is @code{euclidean}. +@end table + @section colormatrix Convert color matrix. diff --git a/libavfilter/Makefile b/libavfilter/Makefile index b870ae2697..1af07108fb 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -218,6 +218,7 @@ OBJS-$(CONFIG_COLORKEY_OPENCL_FILTER) += vf_colorkey_opencl.o opencl.o \ opencl/colorkey.o OBJS-$(CONFIG_COLORHOLD_FILTER) += vf_colorkey.o OBJS-$(CONFIG_COLORLEVELS_FILTER) += vf_colorlevels.o +OBJS-$(CONFIG_COLORMAP_FILTER) += vf_colormap.o OBJS-$(CONFIG_COLORMATRIX_FILTER) += vf_colormatrix.o OBJS-$(CONFIG_COLORSPACE_FILTER) += vf_colorspace.o colorspace.o colorspacedsp.o OBJS-$(CONFIG_COLORTEMPERATURE_FILTER) += vf_colortemperature.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 366c5f4e7f..8c7ae17391 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -204,6 +204,7 @@ extern const AVFilter ff_vf_colorkey; extern const AVFilter ff_vf_colorkey_opencl; extern const AVFilter ff_vf_colorhold; extern const AVFilter ff_vf_colorlevels; +extern const AVFilter ff_vf_colormap; extern const AVFilter ff_vf_colormatrix; extern const AVFilter ff_vf_colorspace; extern const AVFilter ff_vf_colortemperature; diff --git a/libavfilter/version.h b/libavfilter/version.h index 8fccbfc9ec..7b856361c8 100644 --- a/libavfilter/version.h +++ b/libavfilter/version.h @@ -31,7 +31,7 @@ #include "version_major.h" -#define LIBAVFILTER_VERSION_MINOR 33 +#define LIBAVFILTER_VERSION_MINOR 34 #define LIBAVFILTER_VERSION_MICRO 100 diff --git a/libavfilter/vf_colormap.c b/libavfilter/vf_colormap.c new file mode 100644 index 0000000000..7fccc33b4f --- /dev/null +++ b/libavfilter/vf_colormap.c @@ -0,0 +1,582 @@ +/* + * Copyright (c) 2022 Paul B Mahol + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Compute a look-up table from map of colors. + */ + +#include "config_components.h" + +#include "libavutil/attributes.h" +#include "libavutil/avstring.h" +#include "libavutil/common.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" +#include "avfilter.h" +#include "formats.h" +#include "internal.h" +#include "framesync.h" +#include "video.h" + +#define MAX_SIZE 64 + +enum KernelType { + EUCLIDEAN, + WEUCLIDEAN, + NB_KERNELS, +}; + +typedef struct ColorMapContext { + const AVClass *class; + int w, h; + int size; + int nb_maps; + int changed[2]; + + float source[MAX_SIZE][4]; + float ttarget[MAX_SIZE][4]; + float target[MAX_SIZE][4]; + float icoeff[4][4]; + float coeff[MAX_SIZE][4]; + + int target_type; + int kernel_type; + float (*kernel)(const float *x, const float *y); + + FFFrameSync fs; +} ColorMapContext; + +#define OFFSET(x) offsetof(ColorMapContext, x) +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_RUNTIME_PARAM + +static const AVOption colormap_options[] = { + { "patch_size", "set patch size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "64x64"}, 0, 0, FLAGS }, + { "nb_patches", "set number of patches", OFFSET(size), AV_OPT_TYPE_INT, {.i64 = 8}, 1, MAX_SIZE, FLAGS }, + { "type", "set the target type used", OFFSET(target_type), AV_OPT_TYPE_INT, {.i64=1}, 0, 1, FLAGS, "type" }, + { "relative", "the target colors are relative", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 1, FLAGS, "type" }, + { "absolute", "the target colors are absolute", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 1, FLAGS, "type" }, + { "kernel", "set the kernel used for measuring color difference", OFFSET(kernel_type), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_KERNELS-1, FLAGS, "kernel" }, + { "euclidean", "square root of sum of squared differences", 0, AV_OPT_TYPE_CONST, {.i64=EUCLIDEAN}, 0, 0, FLAGS, "kernel" }, + { "weuclidean", "weighted square root of sum of squared differences",0, AV_OPT_TYPE_CONST, {.i64=WEUCLIDEAN}, 0, 0, FLAGS, "kernel" }, + { NULL } +}; + +static int gauss_make_triangular(double *A, int *p, int n) +{ + p[n - 1] = n - 1; + for (int k = 0; k < n; k++) { + double t1; + int m = k; + + for (int i = k + 1; i < n; i++) + if (fabs(A[k + n * i]) > fabs(A[k + n * m])) + m = i; + p[k] = m; + t1 = A[k + n * m]; + A[k + n * m] = A[k + n * k]; + A[k + n * k] = t1; + if (t1 != 0) { + for (int i = k + 1; i < n; i++) + A[k + n * i] /= -t1; + if (k != m) + for (int i = k + 1; i < n; i++) { + double t2 = A[i + n * m]; + A[i + n * m] = A[i + n * k]; + A[i + n * k] = t2; + } + for (int j = k + 1; j < n; j++) + for (int i = k + 1; i < n; i++) + A[i + n * j] += A[k + j * n] * A[i + k * n]; + } else { + return 0; + } + } + + return 1; +} + +static void gauss_solve_triangular(const double *A, const int *p, double *b, int n) +{ + for(int k = 0; k < n - 1; k++) { + int m = p[k]; + double t = b[m]; + b[m] = b[k]; + b[k] = t; + for (int i = k + 1; i < n; i++) + b[i] += A[k + n * i] * t; + } + + for(int k = n - 1; k > 0; k--) { + b[k] /= A[k + n * k]; + double t = b[k]; + for (int i = 0; i < k; i++) + b[i] -= A[k + n * i] * t; + } + + b[0] /= A[0 + 0 * n]; +} + +static int gauss_solve(double *A, double *b, int n) +{ + int *p = av_calloc(n, sizeof(*p)); + + if (!p) + return 1; + + if (!gauss_make_triangular(A, p, n)) { + av_freep(&p); + return 1; + } + + gauss_solve_triangular(A, p, b, n); + + av_freep(&p); + + return 0; +} + +#define P2(x) ((x)*(x)) + +static float euclidean_kernel(const float *x, const float *y) +{ + const float d2 = P2(x[0]-y[0]) + + P2(x[1]-y[1]) + + P2(x[2]-y[2]); + return sqrtf(d2); +} + +static float weuclidean_kernel(const float *x, const float *y) +{ + const float rm = (x[0] + y[0]) * 0.5f; + const float d2 = P2(x[0]-y[0]) * (2.f + rm) + + P2(x[1]-y[1]) * 4.f + + P2(x[2]-y[2]) * (3.f - rm); + return sqrtf(d2); +} + +static void build_map(AVFilterContext *ctx) +{ + ColorMapContext *s = ctx->priv; + + for (int j = 0; j < s->nb_maps; j++) { + s->target[j][0] = s->target_type == 0 ? s->source[j][0] + s->ttarget[j][0] : s->ttarget[j][0]; + s->target[j][1] = s->target_type == 0 ? s->source[j][1] + s->ttarget[j][1] : s->ttarget[j][1]; + s->target[j][2] = s->target_type == 0 ? s->source[j][2] + s->ttarget[j][2] : s->ttarget[j][2]; + } + + for (int c = 0; c < 3; c++) { + for (int j = 0; j < s->nb_maps; j++) + s->coeff[j][c] = 0.f; + + for (int j = 0; j < 4; j++) { + s->icoeff[j][c] = 0; + s->icoeff[j][c] = 0; + s->icoeff[j][c] = 0; + } + + s->icoeff[c+1][c] = 1.f; + + switch (s->nb_maps) { + case 1: + { + float div = fabsf(s->source[0][c]) < 1e-6f ? 1e-6f : s->source[0][c]; + s->icoeff[c][1+c] = s->target[0][c] / div; + } + break; + case 2: + { + double A[2 * 2] = { 1, s->source[0][c], + 1, s->source[1][c] }; + double b[2] = { s->target[0][c], s->target[1][c] }; + + if (gauss_solve(A, b, 2)) + continue; + + s->icoeff[0 ][c] = b[0]; + s->icoeff[1+c][c] = b[1]; + } + break; + case 3: + { + const uint8_t idx[3][3] = {{ 0, 1, 2 }, + { 1, 0, 2 }, + { 2, 0, 1 }}; + const uint8_t didx[3][4] = {{ 0, 1, 2, 2 }, + { 0, 2, 1, 2 }, + { 0, 2, 2, 1 }}; + const int C0 = idx[c][0]; + const int C1 = idx[c][1]; + const int C2 = idx[c][2]; + double A[3 * 3] = { 1, s->source[0][C0], s->source[0][C1] + s->source[0][C2], + 1, s->source[1][C0], s->source[1][C1] + s->source[1][C2], + 1, s->source[2][C0], s->source[2][C1] + s->source[2][C2] }; + double b[3] = { s->target[0][c], s->target[1][c], s->target[2][c] }; + + if (gauss_solve(A, b, 3)) + continue; + + s->icoeff[0][c] = b[didx[c][0]]; + s->icoeff[1][c] = b[didx[c][1]]; + s->icoeff[2][c] = b[didx[c][2]]; + s->icoeff[3][c] = b[didx[c][3]]; + } + break; + case 4: + { + double A[4 * 4] = { 1, s->source[0][0], s->source[0][1], s->source[0][2], + 1, s->source[1][0], s->source[1][1], s->source[1][2], + 1, s->source[2][0], s->source[2][1], s->source[2][2], + 1, s->source[3][0], s->source[3][1], s->source[3][2] }; + double b[4] = { s->target[0][c], s->target[1][c], s->target[2][c], s->target[3][c] }; + int pivot[4]; + + if (!gauss_make_triangular(A, pivot, 4)) + continue; + gauss_solve_triangular(A, pivot, b, 4); + + s->icoeff[0][c] = b[0]; + s->icoeff[1][c] = b[1]; + s->icoeff[2][c] = b[2]; + s->icoeff[3][c] = b[3]; + } + break; + default: + { + const int N = s->nb_maps; + const int N4 = N + 4; + double *A = av_calloc(sizeof(*A), N4 * N4); + double *b = av_calloc(sizeof(*b), N4); + int *pivot = NULL; + + if (!A || !b) + goto error; + + for (int j = 0; j < N; j++) + for (int i = j; i < N; i++) + A[j*N4+i] = A[i*N4+j] = s->kernel(s->source[i], s->source[j]); + + for (int i = 0; i < N; i++) + A[i*N4+N+0] = A[(N+0)*N4+i] = 1; + for (int i = 0; i < N; i++) + A[i*N4+N+1] = A[(N+1)*N4+i] = s->source[i][0]; + for (int i = 0; i < N; i++) + A[i*N4+N+2] = A[(N+2)*N4+i] = s->source[i][1]; + for (int i = 0; i < N; i++) + A[i*N4+N+3] = A[(N+3)*N4+i] = s->source[i][2]; + + for (int j = N; j < N4; j++) + for (int i = N;i < N4; i++) + A[j * N4 + i] = 0.; + + pivot = av_calloc(N4, sizeof(*pivot)); + if (!pivot) + goto error; + + if (gauss_make_triangular(A, pivot, N4)) { + for (int i = 0; i < N; i++) + b[i] = s->target[i][c]; + for (int i = N; i < N + 4; i++) + b[i] = 0; + + gauss_solve_triangular(A, pivot, b, N4); + + for (int i = 0; i < N; i++) + s->coeff[i][c] = b[i]; + + for (int i = 0; i < 4; i++) + s->icoeff[i][c] = b[N + i]; + } +error: + av_free(pivot); + av_free(b); + av_free(A); + } + } + } +} + +typedef struct ThreadData { + AVFrame *in, *out; +} ThreadData; + +static int colormap_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) +{ + ColorMapContext *s = ctx->priv; + ThreadData *td = arg; + AVFrame *in = td->in; + AVFrame *out = td->out; + const int maps = s->nb_maps; + const int width = out->width; + const int height = out->height; + const int slice_start = (height * jobnr) / nb_jobs; + const int slice_end = (height * (jobnr + 1)) / nb_jobs; + const int sr_linesize = in->linesize[2] / 4; + const int dr_linesize = out->linesize[2] / 4; + const int sg_linesize = in->linesize[0] / 4; + const int dg_linesize = out->linesize[0] / 4; + const int sb_linesize = in->linesize[1] / 4; + const int db_linesize = out->linesize[1] / 4; + const float *sr = (float *)in->data[2] + slice_start * sr_linesize; + const float *sg = (float *)in->data[0] + slice_start * sg_linesize; + const float *sb = (float *)in->data[1] + slice_start * sb_linesize; + float *r = (float *)out->data[2] + slice_start * dr_linesize; + float *g = (float *)out->data[0] + slice_start * dg_linesize; + float *b = (float *)out->data[1] + slice_start * db_linesize; + float (*kernel)(const float *x, const float *y) = s->kernel; + const float *icoeff[4] = { s->icoeff[0], s->icoeff[1], s->icoeff[2], s->icoeff[3] }; + + for (int y = slice_start; y < slice_end; y++) { + for (int x = 0; x < width; x++) { + const float input[3] = { sr[x], sg[x], sb[x] }; + float srv, sgv, sbv; + float rv, gv, bv; + + srv = sr[x]; + sgv = sg[x]; + sbv = sb[x]; + + rv = icoeff[0][0]; + gv = icoeff[0][1]; + bv = icoeff[0][2]; + + rv += icoeff[1][0] * srv + icoeff[2][0] * sgv + icoeff[3][0] * sbv; + gv += icoeff[1][1] * srv + icoeff[2][1] * sgv + icoeff[3][1] * sbv; + bv += icoeff[1][2] * srv + icoeff[2][2] * sgv + icoeff[3][2] * sbv; + + for (int z = 0; z < maps && maps > 4; z++) { + const float *coeff = s->coeff[z]; + const float cr = coeff[0]; + const float cg = coeff[1]; + const float cb = coeff[2]; + const float f = kernel(input, s->source[z]); + + rv += f * cr; + gv += f * cg; + bv += f * cb; + } + + r[x] = rv; + g[x] = gv; + b[x] = bv; + } + + sg += sg_linesize; + g += dg_linesize; + sb += sb_linesize; + b += db_linesize; + sr += sr_linesize; + r += dr_linesize; + } + + return 0; +} + +static int import_map(AVFilterLink *inlink, AVFrame *in) +{ + AVFilterContext *ctx = inlink->dst; + ColorMapContext *s = ctx->priv; + const int is_target = FF_INLINK_IDX(inlink) > 1; + const int pw = s->w; + const int pw2 = s->w / 2; + const int ph = s->h; + const int ph2 = s->h / 2; + int changed = 0; + int idx; + + for (int plane = 0; plane < 3; plane++) { + const int c = plane == 0 ? 1 : plane == 1 ? 2 : 0; + + idx = 0; + for (int y = ph2; y < in->height && idx < MAX_SIZE; y += ph) { + const float *src = (const float *)(in->data[plane] + y * in->linesize[plane]); + + for (int x = pw2; x < in->width && idx < MAX_SIZE; x += pw) { + float value = src[x]; + + if (is_target) { + if (s->ttarget[idx][c] != value) + changed = 1; + s->ttarget[idx][c] = value; + } else { + if (s->source[idx][c] != value) + changed = 1; + s->source[idx][c] = value; + } + + idx++; + } + } + } + + if (changed) + s->changed[is_target] = 1; + if (!is_target) + s->nb_maps = FFMIN(idx, s->size); + + return 0; +} + +static int process_frame(FFFrameSync *fs) +{ + AVFilterContext *ctx = fs->parent; + ColorMapContext *s = fs->opaque; + AVFilterLink *outlink = ctx->outputs[0]; + AVFrame *in, *out, *source, *target; + ThreadData td; + int ret; + + switch (s->kernel_type) { + case EUCLIDEAN: + s->kernel = euclidean_kernel; + break; + case WEUCLIDEAN: + s->kernel = weuclidean_kernel; + break; + default: + return AVERROR_BUG; + } + + if ((ret = ff_framesync_get_frame(&s->fs, 0, &in, 1)) < 0 || + (ret = ff_framesync_get_frame(&s->fs, 1, &source, 0)) < 0 || + (ret = ff_framesync_get_frame(&s->fs, 2, &target, 0)) < 0) + return ret; + + import_map(ctx->inputs[1], source); + import_map(ctx->inputs[2], target); + + if (s->changed[0] || s->changed[1]) { + build_map(ctx); + s->changed[0] = s->changed[1] = 0; + } + + if (!ctx->is_disabled) { + if (av_frame_is_writable(in)) { + out = in; + } else { + out = ff_get_video_buffer(outlink, outlink->w, outlink->h); + if (!out) { + av_frame_free(&in); + return AVERROR(ENOMEM); + } + av_frame_copy_props(out, in); + } + + td.in = in; + td.out = out; + ff_filter_execute(ctx, colormap_slice, &td, NULL, + FFMIN(in->height, ff_filter_get_nb_threads(ctx))); + + if (out != in) + av_frame_free(&in); + } else { + out = in; + } + + out->pts = av_rescale_q(s->fs.pts, s->fs.time_base, outlink->time_base); + + return ff_filter_frame(outlink, out); +} + +static int config_output(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + ColorMapContext *s = ctx->priv; + AVFilterLink *inlink = ctx->inputs[0]; + AVFilterLink *source = ctx->inputs[1]; + AVFilterLink *target = ctx->inputs[2]; + FFFrameSyncIn *in; + int ret; + + outlink->time_base = inlink->time_base; + outlink->frame_rate = inlink->frame_rate; + outlink->sample_aspect_ratio = inlink->sample_aspect_ratio; + outlink->w = inlink->w; + outlink->h = inlink->h; + + if ((ret = ff_framesync_init(&s->fs, ctx, 3)) < 0) + return ret; + + in = s->fs.in; + in[0].time_base = inlink->time_base; + in[1].time_base = source->time_base; + in[2].time_base = target->time_base; + in[0].sync = 1; + in[0].before = EXT_STOP; + in[0].after = EXT_INFINITY; + in[1].sync = 1; + in[1].before = EXT_STOP; + in[1].after = EXT_INFINITY; + in[2].sync = 1; + in[2].before = EXT_STOP; + in[2].after = EXT_INFINITY; + s->fs.opaque = s; + s->fs.on_event = process_frame; + + ret = ff_framesync_configure(&s->fs); + outlink->time_base = s->fs.time_base; + + return ret; +} + +static int activate(AVFilterContext *ctx) +{ + ColorMapContext *s = ctx->priv; + return ff_framesync_activate(&s->fs); +} + +static const AVFilterPad inputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + }, + { + .name = "source", + .type = AVMEDIA_TYPE_VIDEO, + }, + { + .name = "target", + .type = AVMEDIA_TYPE_VIDEO, + }, +}; + +static const AVFilterPad outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = config_output, + }, +}; + +AVFILTER_DEFINE_CLASS(colormap); + +const AVFilter ff_vf_colormap = { + .name = "colormap", + .description = NULL_IF_CONFIG_SMALL("Apply custom Color Maps to video stream."), + .priv_class = &colormap_class, + .priv_size = sizeof(ColorMapContext), + .activate = activate, + FILTER_INPUTS(inputs), + FILTER_OUTPUTS(outputs), + FILTER_PIXFMTS(AV_PIX_FMT_GBRPF32, AV_PIX_FMT_GBRAPF32), + .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL | + AVFILTER_FLAG_SLICE_THREADS, + .process_command = ff_filter_process_command, +}; -- cgit v1.2.3