summaryrefslogtreecommitdiff
path: root/libavfilter/dnn
diff options
context:
space:
mode:
Diffstat (limited to 'libavfilter/dnn')
-rw-r--r--libavfilter/dnn/Makefile1
-rw-r--r--libavfilter/dnn/dnn_backend_native.c53
-rw-r--r--libavfilter/dnn/dnn_backend_native.h3
-rw-r--r--libavfilter/dnn/dnn_backend_openvino.c71
-rw-r--r--libavfilter/dnn/dnn_backend_openvino.h2
-rw-r--r--libavfilter/dnn/dnn_backend_tf.c90
-rw-r--r--libavfilter/dnn/dnn_backend_tf.h2
-rw-r--r--libavfilter/dnn/dnn_io_proc.c135
-rw-r--r--libavfilter/dnn/dnn_io_proc.h36
9 files changed, 323 insertions, 70 deletions
diff --git a/libavfilter/dnn/Makefile b/libavfilter/dnn/Makefile
index e0957073ee..ee08cc5243 100644
--- a/libavfilter/dnn/Makefile
+++ b/libavfilter/dnn/Makefile
@@ -1,4 +1,5 @@
OBJS-$(CONFIG_DNN) += dnn/dnn_interface.o
+OBJS-$(CONFIG_DNN) += dnn/dnn_io_proc.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layers.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_avgpool.o
diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c
index 830ec19c80..14e878b6b8 100644
--- a/libavfilter/dnn/dnn_backend_native.c
+++ b/libavfilter/dnn/dnn_backend_native.c
@@ -27,6 +27,7 @@
#include "libavutil/avassert.h"
#include "dnn_backend_native_layer_conv2d.h"
#include "dnn_backend_native_layers.h"
+#include "dnn_io_proc.h"
#define OFFSET(x) offsetof(NativeContext, x)
#define FLAGS AV_OPT_FLAG_FILTERING_PARAM
@@ -69,11 +70,12 @@ static DNNReturnType get_input_native(void *model, DNNData *input, const char *i
return DNN_ERROR;
}
-static DNNReturnType set_input_native(void *model, DNNData *input, const char *input_name)
+static DNNReturnType set_input_native(void *model, AVFrame *frame, const char *input_name)
{
NativeModel *native_model = (NativeModel *)model;
NativeContext *ctx = &native_model->ctx;
DnnOperand *oprd = NULL;
+ DNNData input;
if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
@@ -97,10 +99,8 @@ static DNNReturnType set_input_native(void *model, DNNData *input, const char *i
return DNN_ERROR;
}
- oprd->dims[0] = 1;
- oprd->dims[1] = input->height;
- oprd->dims[2] = input->width;
- oprd->dims[3] = input->channels;
+ oprd->dims[1] = frame->height;
+ oprd->dims[2] = frame->width;
av_freep(&oprd->data);
oprd->length = calculate_operand_data_length(oprd);
@@ -114,7 +114,16 @@ static DNNReturnType set_input_native(void *model, DNNData *input, const char *i
return DNN_ERROR;
}
- input->data = oprd->data;
+ input.height = oprd->dims[1];
+ input.width = oprd->dims[2];
+ input.channels = oprd->dims[3];
+ input.data = oprd->data;
+ input.dt = oprd->data_type;
+ if (native_model->model->pre_proc != NULL) {
+ native_model->model->pre_proc(frame, &input, native_model->model->userdata);
+ } else {
+ proc_from_frame_to_dnn(frame, &input, ctx);
+ }
return DNN_SUCCESS;
}
@@ -185,6 +194,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *optio
if (av_opt_set_from_string(&native_model->ctx, model->options, NULL, "=", "&") < 0)
goto fail;
model->model = (void *)native_model;
+ native_model->model = model;
#if !HAVE_PTHREAD_CANCEL
if (native_model->ctx.options.conv2d_threads > 1){
@@ -275,11 +285,19 @@ fail:
return NULL;
}
-DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, const char **output_names, uint32_t nb_output)
+DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, const char **output_names, uint32_t nb_output, AVFrame *out_frame)
{
NativeModel *native_model = (NativeModel *)model->model;
NativeContext *ctx = &native_model->ctx;
int32_t layer;
+ DNNData output;
+
+ if (nb_output != 1) {
+ // currently, the filter does not need multiple outputs,
+ // so we just pending the support until we really need it.
+ av_log(ctx, AV_LOG_ERROR, "do not support multiple outputs\n");
+ return DNN_ERROR;
+ }
if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
@@ -317,11 +335,22 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
return DNN_ERROR;
}
- outputs[i].data = oprd->data;
- outputs[i].height = oprd->dims[1];
- outputs[i].width = oprd->dims[2];
- outputs[i].channels = oprd->dims[3];
- outputs[i].dt = oprd->data_type;
+ output.data = oprd->data;
+ output.height = oprd->dims[1];
+ output.width = oprd->dims[2];
+ output.channels = oprd->dims[3];
+ output.dt = oprd->data_type;
+
+ if (out_frame->width != output.width || out_frame->height != output.height) {
+ out_frame->width = output.width;
+ out_frame->height = output.height;
+ } else {
+ if (native_model->model->post_proc != NULL) {
+ native_model->model->post_proc(out_frame, &output, native_model->model->userdata);
+ } else {
+ proc_from_dnn_to_frame(out_frame, &output, ctx);
+ }
+ }
}
return DNN_SUCCESS;
diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h
index 33634118a8..553438bd22 100644
--- a/libavfilter/dnn/dnn_backend_native.h
+++ b/libavfilter/dnn/dnn_backend_native.h
@@ -119,6 +119,7 @@ typedef struct NativeContext {
// Represents simple feed-forward convolutional network.
typedef struct NativeModel{
NativeContext ctx;
+ DNNModel *model;
Layer *layers;
int32_t layers_num;
DnnOperand *operands;
@@ -127,7 +128,7 @@ typedef struct NativeModel{
DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *options, void *userdata);
-DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, const char **output_names, uint32_t nb_output);
+DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, const char **output_names, uint32_t nb_output, AVFrame *out_frame);
void ff_dnn_free_model_native(DNNModel **model);
diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c
index 01e1a1d4c8..b1bad3f659 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -24,6 +24,7 @@
*/
#include "dnn_backend_openvino.h"
+#include "dnn_io_proc.h"
#include "libavformat/avio.h"
#include "libavutil/avassert.h"
#include "libavutil/opt.h"
@@ -42,6 +43,7 @@ typedef struct OVContext {
typedef struct OVModel{
OVContext ctx;
+ DNNModel *model;
ie_core_t *core;
ie_network_t *network;
ie_executable_network_t *exe_network;
@@ -131,7 +133,7 @@ static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input
return DNN_ERROR;
}
-static DNNReturnType set_input_ov(void *model, DNNData *input, const char *input_name)
+static DNNReturnType set_input_ov(void *model, AVFrame *frame, const char *input_name)
{
OVModel *ov_model = (OVModel *)model;
OVContext *ctx = &ov_model->ctx;
@@ -139,10 +141,7 @@ static DNNReturnType set_input_ov(void *model, DNNData *input, const char *input
dimensions_t dims;
precision_e precision;
ie_blob_buffer_t blob_buffer;
-
- status = ie_exec_network_create_infer_request(ov_model->exe_network, &ov_model->infer_request);
- if (status != OK)
- goto err;
+ DNNData input;
status = ie_infer_request_get_blob(ov_model->infer_request, input_name, &ov_model->input_blob);
if (status != OK)
@@ -153,23 +152,26 @@ static DNNReturnType set_input_ov(void *model, DNNData *input, const char *input
if (status != OK)
goto err;
- av_assert0(input->channels == dims.dims[1]);
- av_assert0(input->height == dims.dims[2]);
- av_assert0(input->width == dims.dims[3]);
- av_assert0(input->dt == precision_to_datatype(precision));
-
status = ie_blob_get_buffer(ov_model->input_blob, &blob_buffer);
if (status != OK)
goto err;
- input->data = blob_buffer.buffer;
+
+ input.height = dims.dims[2];
+ input.width = dims.dims[3];
+ input.channels = dims.dims[1];
+ input.data = blob_buffer.buffer;
+ input.dt = precision_to_datatype(precision);
+ if (ov_model->model->pre_proc != NULL) {
+ ov_model->model->pre_proc(frame, &input, ov_model->model->userdata);
+ } else {
+ proc_from_frame_to_dnn(frame, &input, ctx);
+ }
return DNN_SUCCESS;
err:
if (ov_model->input_blob)
ie_blob_free(&ov_model->input_blob);
- if (ov_model->infer_request)
- ie_infer_request_free(&ov_model->infer_request);
av_log(ctx, AV_LOG_ERROR, "Failed to create inference instance or get input data/dims/precision/memory\n");
return DNN_ERROR;
}
@@ -184,7 +186,7 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options,
ie_config_t config = {NULL, NULL, NULL};
ie_available_devices_t a_dev;
- model = av_malloc(sizeof(DNNModel));
+ model = av_mallocz(sizeof(DNNModel));
if (!model){
return NULL;
}
@@ -192,6 +194,7 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options,
ov_model = av_mallocz(sizeof(OVModel));
if (!ov_model)
goto err;
+ ov_model->model = model;
ov_model->ctx.class = &dnn_openvino_class;
ctx = &ov_model->ctx;
@@ -226,6 +229,10 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options,
goto err;
}
+ status = ie_exec_network_create_infer_request(ov_model->exe_network, &ov_model->infer_request);
+ if (status != OK)
+ goto err;
+
model->model = (void *)ov_model;
model->set_input = &set_input_ov;
model->get_input = &get_input_ov;
@@ -238,6 +245,8 @@ err:
if (model)
av_freep(&model);
if (ov_model) {
+ if (ov_model->infer_request)
+ ie_infer_request_free(&ov_model->infer_request);
if (ov_model->exe_network)
ie_exec_network_free(&ov_model->exe_network);
if (ov_model->network)
@@ -249,7 +258,7 @@ err:
return NULL;
}
-DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, const char **output_names, uint32_t nb_output)
+DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char **output_names, uint32_t nb_output, AVFrame *out_frame)
{
char *model_output_name = NULL;
char *all_output_names = NULL;
@@ -258,8 +267,18 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, c
ie_blob_buffer_t blob_buffer;
OVModel *ov_model = (OVModel *)model->model;
OVContext *ctx = &ov_model->ctx;
- IEStatusCode status = ie_infer_request_infer(ov_model->infer_request);
+ IEStatusCode status;
size_t model_output_count = 0;
+ DNNData output;
+
+ if (nb_output != 1) {
+ // currently, the filter does not need multiple outputs,
+ // so we just pending the support until we really need it.
+ av_log(ctx, AV_LOG_ERROR, "do not support multiple outputs\n");
+ return DNN_ERROR;
+ }
+
+ status = ie_infer_request_infer(ov_model->infer_request);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to start synchronous model inference\n");
return DNN_ERROR;
@@ -296,11 +315,21 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, c
return DNN_ERROR;
}
- outputs[i].channels = dims.dims[1];
- outputs[i].height = dims.dims[2];
- outputs[i].width = dims.dims[3];
- outputs[i].dt = precision_to_datatype(precision);
- outputs[i].data = blob_buffer.buffer;
+ output.channels = dims.dims[1];
+ output.height = dims.dims[2];
+ output.width = dims.dims[3];
+ output.dt = precision_to_datatype(precision);
+ output.data = blob_buffer.buffer;
+ if (out_frame->width != output.width || out_frame->height != output.height) {
+ out_frame->width = output.width;
+ out_frame->height = output.height;
+ } else {
+ if (ov_model->model->post_proc != NULL) {
+ ov_model->model->post_proc(out_frame, &output, ov_model->model->userdata);
+ } else {
+ proc_from_dnn_to_frame(out_frame, &output, ctx);
+ }
+ }
}
return DNN_SUCCESS;
diff --git a/libavfilter/dnn/dnn_backend_openvino.h b/libavfilter/dnn/dnn_backend_openvino.h
index f69bc5ca0c..efb349cb49 100644
--- a/libavfilter/dnn/dnn_backend_openvino.h
+++ b/libavfilter/dnn/dnn_backend_openvino.h
@@ -31,7 +31,7 @@
DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options, void *userdata);
-DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, const char **output_names, uint32_t nb_output);
+DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char **output_names, uint32_t nb_output, AVFrame *out_frame);
void ff_dnn_free_model_ov(DNNModel **model);
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index bac7d8c420..c2d8c06931 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -31,6 +31,7 @@
#include "libavutil/avassert.h"
#include "dnn_backend_native_layer_pad.h"
#include "dnn_backend_native_layer_maximum.h"
+#include "dnn_io_proc.h"
#include <tensorflow/c/c_api.h>
@@ -40,13 +41,12 @@ typedef struct TFContext {
typedef struct TFModel{
TFContext ctx;
+ DNNModel *model;
TF_Graph *graph;
TF_Session *session;
TF_Status *status;
TF_Output input;
TF_Tensor *input_tensor;
- TF_Tensor **output_tensors;
- uint32_t nb_output;
} TFModel;
static const AVClass dnn_tensorflow_class = {
@@ -152,13 +152,19 @@ static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input
return DNN_SUCCESS;
}
-static DNNReturnType set_input_tf(void *model, DNNData *input, const char *input_name)
+static DNNReturnType set_input_tf(void *model, AVFrame *frame, const char *input_name)
{
TFModel *tf_model = (TFModel *)model;
TFContext *ctx = &tf_model->ctx;
+ DNNData input;
TF_SessionOptions *sess_opts;
const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init");
+ if (get_input_tf(model, &input, input_name) != DNN_SUCCESS)
+ return DNN_ERROR;
+ input.height = frame->height;
+ input.width = frame->width;
+
// Input operation
tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name);
if (!tf_model->input.oper){
@@ -169,12 +175,18 @@ static DNNReturnType set_input_tf(void *model, DNNData *input, const char *input
if (tf_model->input_tensor){
TF_DeleteTensor(tf_model->input_tensor);
}
- tf_model->input_tensor = allocate_input_tensor(input);
+ tf_model->input_tensor = allocate_input_tensor(&input);
if (!tf_model->input_tensor){
av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
return DNN_ERROR;
}
- input->data = (float *)TF_TensorData(tf_model->input_tensor);
+ input.data = (float *)TF_TensorData(tf_model->input_tensor);
+
+ if (tf_model->model->pre_proc != NULL) {
+ tf_model->model->pre_proc(frame, &input, tf_model->model->userdata);
+ } else {
+ proc_from_frame_to_dnn(frame, &input, ctx);
+ }
// session
if (tf_model->session){
@@ -591,7 +603,7 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options,
DNNModel *model = NULL;
TFModel *tf_model = NULL;
- model = av_malloc(sizeof(DNNModel));
+ model = av_mallocz(sizeof(DNNModel));
if (!model){
return NULL;
}
@@ -602,6 +614,7 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options,
return NULL;
}
tf_model->ctx.class = &dnn_tensorflow_class;
+ tf_model->model = model;
if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){
if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){
@@ -621,11 +634,20 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options,
return model;
}
-DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, const char **output_names, uint32_t nb_output)
+DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, const char **output_names, uint32_t nb_output, AVFrame *out_frame)
{
TF_Output *tf_outputs;
TFModel *tf_model = (TFModel *)model->model;
TFContext *ctx = &tf_model->ctx;
+ DNNData output;
+ TF_Tensor **output_tensors;
+
+ if (nb_output != 1) {
+ // currently, the filter does not need multiple outputs,
+ // so we just pending the support until we really need it.
+ av_log(ctx, AV_LOG_ERROR, "do not support multiple outputs\n");
+ return DNN_ERROR;
+ }
tf_outputs = av_malloc_array(nb_output, sizeof(*tf_outputs));
if (tf_outputs == NULL) {
@@ -633,18 +655,8 @@ DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, c
return DNN_ERROR;
}
- if (tf_model->output_tensors) {
- for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
- if (tf_model->output_tensors[i]) {
- TF_DeleteTensor(tf_model->output_tensors[i]);
- tf_model->output_tensors[i] = NULL;
- }
- }
- }
- av_freep(&tf_model->output_tensors);
- tf_model->nb_output = nb_output;
- tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors));
- if (!tf_model->output_tensors) {
+ output_tensors = av_mallocz_array(nb_output, sizeof(*output_tensors));
+ if (!output_tensors) {
av_freep(&tf_outputs);
av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output tensor\n"); \
return DNN_ERROR;
@@ -654,6 +666,7 @@ DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, c
tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]);
if (!tf_outputs[i].oper) {
av_freep(&tf_outputs);
+ av_freep(&output_tensors);
av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", output_names[i]); \
return DNN_ERROR;
}
@@ -662,22 +675,40 @@ DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, c
TF_SessionRun(tf_model->session, NULL,
&tf_model->input, &tf_model->input_tensor, 1,
- tf_outputs, tf_model->output_tensors, nb_output,
+ tf_outputs, output_tensors, nb_output,
NULL, 0, NULL, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK) {
av_freep(&tf_outputs);
+ av_freep(&output_tensors);
av_log(ctx, AV_LOG_ERROR, "Failed to run session when executing model\n");
return DNN_ERROR;
}
for (uint32_t i = 0; i < nb_output; ++i) {
- outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1);
- outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2);
- outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3);
- outputs[i].data = TF_TensorData(tf_model->output_tensors[i]);
- outputs[i].dt = TF_TensorType(tf_model->output_tensors[i]);
+ output.height = TF_Dim(output_tensors[i], 1);
+ output.width = TF_Dim(output_tensors[i], 2);
+ output.channels = TF_Dim(output_tensors[i], 3);
+ output.data = TF_TensorData(output_tensors[i]);
+ output.dt = TF_TensorType(output_tensors[i]);
+
+ if (out_frame->width != output.width || out_frame->height != output.height) {
+ out_frame->width = output.width;
+ out_frame->height = output.height;
+ } else {
+ if (tf_model->model->post_proc != NULL) {
+ tf_model->model->post_proc(out_frame, &output, tf_model->model->userdata);
+ } else {
+ proc_from_dnn_to_frame(out_frame, &output, ctx);
+ }
+ }
}
+ for (uint32_t i = 0; i < nb_output; ++i) {
+ if (output_tensors[i]) {
+ TF_DeleteTensor(output_tensors[i]);
+ }
+ }
+ av_freep(&output_tensors);
av_freep(&tf_outputs);
return DNN_SUCCESS;
}
@@ -701,15 +732,6 @@ void ff_dnn_free_model_tf(DNNModel **model)
if (tf_model->input_tensor){
TF_DeleteTensor(tf_model->input_tensor);
}
- if (tf_model->output_tensors) {
- for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
- if (tf_model->output_tensors[i]) {
- TF_DeleteTensor(tf_model->output_tensors[i]);
- tf_model->output_tensors[i] = NULL;
- }
- }
- }
- av_freep(&tf_model->output_tensors);
av_freep(&tf_model);
av_freep(model);
}
diff --git a/libavfilter/dnn/dnn_backend_tf.h b/libavfilter/dnn/dnn_backend_tf.h
index 1cf5cc9e76..f379e83d8d 100644
--- a/libavfilter/dnn/dnn_backend_tf.h
+++ b/libavfilter/dnn/dnn_backend_tf.h
@@ -31,7 +31,7 @@
DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options, void *userdata);
-DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, const char **output_names, uint32_t nb_output);
+DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, const char **output_names, uint32_t nb_output, AVFrame *out_frame);
void ff_dnn_free_model_tf(DNNModel **model);
diff --git a/libavfilter/dnn/dnn_io_proc.c b/libavfilter/dnn/dnn_io_proc.c
new file mode 100644
index 0000000000..8ce1959b42
--- /dev/null
+++ b/libavfilter/dnn/dnn_io_proc.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2020
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "dnn_io_proc.h"
+#include "libavutil/imgutils.h"
+#include "libswscale/swscale.h"
+
+DNNReturnType proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
+{
+ struct SwsContext *sws_ctx;
+ int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
+ if (output->dt != DNN_FLOAT) {
+ av_log(log_ctx, AV_LOG_ERROR, "do not support data type rather than DNN_FLOAT\n");
+ return DNN_ERROR;
+ }
+
+ switch (frame->format) {
+ case AV_PIX_FMT_RGB24:
+ case AV_PIX_FMT_BGR24:
+ sws_ctx = sws_getContext(frame->width * 3,
+ frame->height,
+ AV_PIX_FMT_GRAYF32,
+ frame->width * 3,
+ frame->height,
+ AV_PIX_FMT_GRAY8,
+ 0, NULL, NULL, NULL);
+ sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0},
+ (const int[4]){frame->width * 3 * sizeof(float), 0, 0, 0}, 0, frame->height,
+ (uint8_t * const*)frame->data, frame->linesize);
+ sws_freeContext(sws_ctx);
+ return DNN_SUCCESS;
+ case AV_PIX_FMT_GRAYF32:
+ av_image_copy_plane(frame->data[0], frame->linesize[0],
+ output->data, bytewidth,
+ bytewidth, frame->height);
+ return DNN_SUCCESS;
+ case AV_PIX_FMT_YUV420P:
+ case AV_PIX_FMT_YUV422P:
+ case AV_PIX_FMT_YUV444P:
+ case AV_PIX_FMT_YUV410P:
+ case AV_PIX_FMT_YUV411P:
+ case AV_PIX_FMT_GRAY8:
+ sws_ctx = sws_getContext(frame->width,
+ frame->height,
+ AV_PIX_FMT_GRAYF32,
+ frame->width,
+ frame->height,
+ AV_PIX_FMT_GRAY8,
+ 0, NULL, NULL, NULL);
+ sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0},
+ (const int[4]){frame->width * sizeof(float), 0, 0, 0}, 0, frame->height,
+ (uint8_t * const*)frame->data, frame->linesize);
+ sws_freeContext(sws_ctx);
+ return DNN_SUCCESS;
+ default:
+ av_log(log_ctx, AV_LOG_ERROR, "do not support frame format %d\n", frame->format);
+ return DNN_ERROR;
+ }
+
+ return DNN_SUCCESS;
+}
+
+DNNReturnType proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
+{
+ struct SwsContext *sws_ctx;
+ int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
+ if (input->dt != DNN_FLOAT) {
+ av_log(log_ctx, AV_LOG_ERROR, "do not support data type rather than DNN_FLOAT\n");
+ return DNN_ERROR;
+ }
+
+ switch (frame->format) {
+ case AV_PIX_FMT_RGB24:
+ case AV_PIX_FMT_BGR24:
+ sws_ctx = sws_getContext(frame->width * 3,
+ frame->height,
+ AV_PIX_FMT_GRAY8,
+ frame->width * 3,
+ frame->height,
+ AV_PIX_FMT_GRAYF32,
+ 0, NULL, NULL, NULL);
+ sws_scale(sws_ctx, (const uint8_t **)frame->data,
+ frame->linesize, 0, frame->height,
+ (uint8_t * const*)(&input->data),
+ (const int [4]){frame->width * 3 * sizeof(float), 0, 0, 0});
+ sws_freeContext(sws_ctx);
+ break;
+ case AV_PIX_FMT_GRAYF32:
+ av_image_copy_plane(input->data, bytewidth,
+ frame->data[0], frame->linesize[0],
+ bytewidth, frame->height);
+ break;
+ case AV_PIX_FMT_YUV420P:
+ case AV_PIX_FMT_YUV422P:
+ case AV_PIX_FMT_YUV444P:
+ case AV_PIX_FMT_YUV410P:
+ case AV_PIX_FMT_YUV411P:
+ case AV_PIX_FMT_GRAY8:
+ sws_ctx = sws_getContext(frame->width,
+ frame->height,
+ AV_PIX_FMT_GRAY8,
+ frame->width,
+ frame->height,
+ AV_PIX_FMT_GRAYF32,
+ 0, NULL, NULL, NULL);
+ sws_scale(sws_ctx, (const uint8_t **)frame->data,
+ frame->linesize, 0, frame->height,
+ (uint8_t * const*)(&input->data),
+ (const int [4]){frame->width * sizeof(float), 0, 0, 0});
+ sws_freeContext(sws_ctx);
+ break;
+ default:
+ av_log(log_ctx, AV_LOG_ERROR, "do not support frame format %d\n", frame->format);
+ return DNN_ERROR;
+ }
+
+ return DNN_SUCCESS;
+}
diff --git a/libavfilter/dnn/dnn_io_proc.h b/libavfilter/dnn/dnn_io_proc.h
new file mode 100644
index 0000000000..4c7dc7c1a2
--- /dev/null
+++ b/libavfilter/dnn/dnn_io_proc.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2020
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * DNN input&output process between AVFrame and DNNData.
+ */
+
+
+#ifndef AVFILTER_DNN_DNN_IO_PROC_H
+#define AVFILTER_DNN_DNN_IO_PROC_H
+
+#include "../dnn_interface.h"
+#include "libavutil/frame.h"
+
+DNNReturnType proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx);
+DNNReturnType proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx);
+
+#endif
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-23 12:43:18 +0000