summaryrefslogtreecommitdiff
path: root/libavfilter/dnn
diff options
context:
space:
mode:
Diffstat (limited to 'libavfilter/dnn')
-rw-r--r--libavfilter/dnn/dnn_backend_common.c4
-rw-r--r--libavfilter/dnn/dnn_backend_common.h5
-rw-r--r--libavfilter/dnn/dnn_backend_native.c59
-rw-r--r--libavfilter/dnn/dnn_backend_native.h6
-rw-r--r--libavfilter/dnn/dnn_backend_openvino.c99
-rw-r--r--libavfilter/dnn/dnn_backend_openvino.h3
-rw-r--r--libavfilter/dnn/dnn_backend_tf.c37
-rw-r--r--libavfilter/dnn/dnn_backend_tf.h3
-rw-r--r--libavfilter/dnn/dnn_interface.c8
9 files changed, 115 insertions, 109 deletions
diff --git a/libavfilter/dnn/dnn_backend_common.c b/libavfilter/dnn/dnn_backend_common.c
index 426683b73d..6a9c4cc87f 100644
--- a/libavfilter/dnn/dnn_backend_common.c
+++ b/libavfilter/dnn/dnn_backend_common.c
@@ -38,7 +38,7 @@ int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func
return AVERROR(EINVAL);
}
- if (!exec_params->out_frame) {
+ if (!exec_params->out_frame && func_type == DFT_PROCESS_FRAME) {
av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
return AVERROR(EINVAL);
}
@@ -138,7 +138,7 @@ DNNReturnType ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_
return DNN_SUCCESS;
}
-DNNAsyncStatusType ff_dnn_get_async_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
+DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
{
TaskItem *task = ff_queue_peek_front(task_queue);
diff --git a/libavfilter/dnn/dnn_backend_common.h b/libavfilter/dnn/dnn_backend_common.h
index 604c1d3bd7..78e62a94a2 100644
--- a/libavfilter/dnn/dnn_backend_common.h
+++ b/libavfilter/dnn/dnn_backend_common.h
@@ -29,7 +29,8 @@
#include "libavutil/thread.h"
#define DNN_BACKEND_COMMON_OPTIONS \
- { "nireq", "number of request", OFFSET(options.nireq), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS },
+ { "nireq", "number of request", OFFSET(options.nireq), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, \
+ { "async", "use DNN async inference", OFFSET(options.async), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, FLAGS },
// one task for one function call from dnn interface
typedef struct TaskItem {
@@ -135,7 +136,7 @@ DNNReturnType ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_
* @retval DAST_NOT_READY if inference not completed yet.
* @retval DAST_SUCCESS if result successfully extracted
*/
-DNNAsyncStatusType ff_dnn_get_async_result_common(Queue *task_queue, AVFrame **in, AVFrame **out);
+DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out);
/**
* Allocate input and output frames and fill the Task
diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c
index 3b2a3aa55d..2d34b88f8a 100644
--- a/libavfilter/dnn/dnn_backend_native.c
+++ b/libavfilter/dnn/dnn_backend_native.c
@@ -34,6 +34,7 @@
#define FLAGS AV_OPT_FLAG_FILTERING_PARAM
static const AVOption dnn_native_options[] = {
{ "conv2d_threads", "threads num for conv2d layer", OFFSET(options.conv2d_threads), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS },
+ { "async", "use DNN async inference", OFFSET(options.async), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
{ NULL },
};
@@ -189,6 +190,11 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType f
goto fail;
native_model->model = model;
+ if (native_model->ctx.options.async) {
+ av_log(&native_model->ctx, AV_LOG_WARNING, "Async not supported. Rolling back to sync\n");
+ native_model->ctx.options.async = 0;
+ }
+
#if !HAVE_PTHREAD_CANCEL
if (native_model->ctx.options.conv2d_threads > 1){
av_log(&native_model->ctx, AV_LOG_WARNING, "'conv2d_threads' option was set but it is not supported "
@@ -212,6 +218,11 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType f
goto fail;
}
+ native_model->task_queue = ff_queue_create();
+ if (!native_model->task_queue) {
+ goto fail;
+ }
+
native_model->inference_queue = ff_queue_create();
if (!native_model->inference_queue) {
goto fail;
@@ -425,17 +436,30 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNExecBasePara
{
NativeModel *native_model = model->model;
NativeContext *ctx = &native_model->ctx;
- TaskItem task;
+ TaskItem *task;
if (ff_check_exec_params(ctx, DNN_NATIVE, model->func_type, exec_params) != 0) {
return DNN_ERROR;
}
- if (ff_dnn_fill_task(&task, exec_params, native_model, 0, 1) != DNN_SUCCESS) {
+ task = av_malloc(sizeof(*task));
+ if (!task) {
+ av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
return DNN_ERROR;
}
- if (extract_inference_from_task(&task, native_model->inference_queue) != DNN_SUCCESS) {
+ if (ff_dnn_fill_task(task, exec_params, native_model, ctx->options.async, 1) != DNN_SUCCESS) {
+ av_freep(&task);
+ return DNN_ERROR;
+ }
+
+ if (ff_queue_push_back(native_model->task_queue, task) < 0) {
+ av_freep(&task);
+ av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
+ return DNN_ERROR;
+ }
+
+ if (extract_inference_from_task(task, native_model->inference_queue) != DNN_SUCCESS) {
av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
return DNN_ERROR;
}
@@ -443,6 +467,26 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNExecBasePara
return execute_model_native(native_model->inference_queue);
}
+DNNReturnType ff_dnn_flush_native(const DNNModel *model)
+{
+ NativeModel *native_model = model->model;
+
+ if (ff_queue_size(native_model->inference_queue) == 0) {
+ // no pending task need to flush
+ return DNN_SUCCESS;
+ }
+
+ // for now, use sync node with flush operation
+ // Switch to async when it is supported
+ return execute_model_native(native_model->inference_queue);
+}
+
+DNNAsyncStatusType ff_dnn_get_result_native(const DNNModel *model, AVFrame **in, AVFrame **out)
+{
+ NativeModel *native_model = model->model;
+ return ff_dnn_get_result_common(native_model->task_queue, in, out);
+}
+
int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd)
{
int32_t result = 1;
@@ -497,6 +541,15 @@ void ff_dnn_free_model_native(DNNModel **model)
av_freep(&item);
}
ff_queue_destroy(native_model->inference_queue);
+
+ while (ff_queue_size(native_model->task_queue) != 0) {
+ TaskItem *item = ff_queue_pop_front(native_model->task_queue);
+ av_frame_free(&item->in_frame);
+ av_frame_free(&item->out_frame);
+ av_freep(&item);
+ }
+ ff_queue_destroy(native_model->task_queue);
+
av_freep(&native_model);
}
av_freep(model);
diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h
index 1b9d5bdf2d..ca61bb353f 100644
--- a/libavfilter/dnn/dnn_backend_native.h
+++ b/libavfilter/dnn/dnn_backend_native.h
@@ -111,6 +111,7 @@ typedef struct InputParams{
} InputParams;
typedef struct NativeOptions{
+ uint8_t async;
uint32_t conv2d_threads;
} NativeOptions;
@@ -127,6 +128,7 @@ typedef struct NativeModel{
int32_t layers_num;
DnnOperand *operands;
int32_t operands_num;
+ Queue *task_queue;
Queue *inference_queue;
} NativeModel;
@@ -134,6 +136,10 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType f
DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params);
+DNNAsyncStatusType ff_dnn_get_result_native(const DNNModel *model, AVFrame **in, AVFrame **out);
+
+DNNReturnType ff_dnn_flush_native(const DNNModel *model);
+
void ff_dnn_free_model_native(DNNModel **model);
// NOTE: User must check for error (return value <= 0) to handle
diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c
index c825e70c82..bf13b017fb 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -39,6 +39,7 @@
typedef struct OVOptions{
char *device_type;
int nireq;
+ uint8_t async;
int batch_size;
int input_resizable;
} OVOptions;
@@ -271,14 +272,14 @@ static void infer_completion_callback(void *args)
av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post proc\n");
return;
}
- ov_model->model->detect_post_proc(task->out_frame, &output, 1, ov_model->model->filter_ctx);
+ ov_model->model->detect_post_proc(task->in_frame, &output, 1, ov_model->model->filter_ctx);
break;
case DFT_ANALYTICS_CLASSIFY:
if (!ov_model->model->classify_post_proc) {
av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide post proc\n");
return;
}
- ov_model->model->classify_post_proc(task->out_frame, &output, request->inferences[i]->bbox_index, ov_model->model->filter_ctx);
+ ov_model->model->classify_post_proc(task->in_frame, &output, request->inferences[i]->bbox_index, ov_model->model->filter_ctx);
break;
default:
av_assert0(!"should not reach here");
@@ -761,55 +762,6 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams *
{
OVModel *ov_model = model->model;
OVContext *ctx = &ov_model->ctx;
- TaskItem task;
- OVRequestItem *request;
-
- if (ff_check_exec_params(ctx, DNN_OV, model->func_type, exec_params) != 0) {
- return DNN_ERROR;
- }
-
- if (model->func_type == DFT_ANALYTICS_CLASSIFY) {
- // Once we add async support for tensorflow backend and native backend,
- // we'll combine the two sync/async functions in dnn_interface.h to
- // simplify the code in filter, and async will be an option within backends.
- // so, do not support now, and classify filter will not call this function.
- return DNN_ERROR;
- }
-
- if (ctx->options.batch_size > 1) {
- avpriv_report_missing_feature(ctx, "batch mode for sync execution");
- return DNN_ERROR;
- }
-
- if (!ov_model->exe_network) {
- if (init_model_ov(ov_model, exec_params->input_name, exec_params->output_names[0]) != DNN_SUCCESS) {
- av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
- return DNN_ERROR;
- }
- }
-
- if (ff_dnn_fill_task(&task, exec_params, ov_model, 0, 1) != DNN_SUCCESS) {
- return DNN_ERROR;
- }
-
- if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue, exec_params) != DNN_SUCCESS) {
- av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
- return DNN_ERROR;
- }
-
- request = ff_safe_queue_pop_front(ov_model->request_queue);
- if (!request) {
- av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
- return DNN_ERROR;
- }
-
- return execute_model_ov(request, ov_model->inference_queue);
-}
-
-DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBaseParams *exec_params)
-{
- OVModel *ov_model = model->model;
- OVContext *ctx = &ov_model->ctx;
OVRequestItem *request;
TaskItem *task;
DNNReturnType ret;
@@ -831,7 +783,8 @@ DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBasePa
return DNN_ERROR;
}
- if (ff_dnn_fill_task(task, exec_params, ov_model, 1, 1) != DNN_SUCCESS) {
+ if (ff_dnn_fill_task(task, exec_params, ov_model, ctx->options.async, 1) != DNN_SUCCESS) {
+ av_freep(&task);
return DNN_ERROR;
}
@@ -846,26 +799,48 @@ DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBasePa
return DNN_ERROR;
}
- while (ff_queue_size(ov_model->inference_queue) >= ctx->options.batch_size) {
+ if (ctx->options.async) {
+ while (ff_queue_size(ov_model->inference_queue) >= ctx->options.batch_size) {
+ request = ff_safe_queue_pop_front(ov_model->request_queue);
+ if (!request) {
+ av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
+ return DNN_ERROR;
+ }
+
+ ret = execute_model_ov(request, ov_model->inference_queue);
+ if (ret != DNN_SUCCESS) {
+ return ret;
+ }
+ }
+
+ return DNN_SUCCESS;
+ }
+ else {
+ if (model->func_type == DFT_ANALYTICS_CLASSIFY) {
+ // Classification filter has not been completely
+ // tested with the sync mode. So, do not support now.
+ avpriv_report_missing_feature(ctx, "classify for sync execution");
+ return DNN_ERROR;
+ }
+
+ if (ctx->options.batch_size > 1) {
+ avpriv_report_missing_feature(ctx, "batch mode for sync execution");
+ return DNN_ERROR;
+ }
+
request = ff_safe_queue_pop_front(ov_model->request_queue);
if (!request) {
av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
return DNN_ERROR;
}
-
- ret = execute_model_ov(request, ov_model->inference_queue);
- if (ret != DNN_SUCCESS) {
- return ret;
- }
+ return execute_model_ov(request, ov_model->inference_queue);
}
-
- return DNN_SUCCESS;
}
-DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out)
+DNNAsyncStatusType ff_dnn_get_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out)
{
OVModel *ov_model = model->model;
- return ff_dnn_get_async_result_common(ov_model->task_queue, in, out);
+ return ff_dnn_get_result_common(ov_model->task_queue, in, out);
}
DNNReturnType ff_dnn_flush_ov(const DNNModel *model)
diff --git a/libavfilter/dnn/dnn_backend_openvino.h b/libavfilter/dnn/dnn_backend_openvino.h
index 046d0c5b5a..0bbca0c057 100644
--- a/libavfilter/dnn/dnn_backend_openvino.h
+++ b/libavfilter/dnn/dnn_backend_openvino.h
@@ -32,8 +32,7 @@
DNNModel *ff_dnn_load_model_ov(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams *exec_params);
-DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBaseParams *exec_params);
-DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out);
+DNNAsyncStatusType ff_dnn_get_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out);
DNNReturnType ff_dnn_flush_ov(const DNNModel *model);
void ff_dnn_free_model_ov(DNNModel **model);
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index ffec1b1328..84952ece5a 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -42,6 +42,7 @@
typedef struct TFOptions{
char *sess_config;
+ uint8_t async;
uint32_t nireq;
} TFOptions;
@@ -1061,7 +1062,7 @@ static void infer_completion_callback(void *args) {
av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n");
return;
}
- tf_model->model->detect_post_proc(task->out_frame, outputs, task->nb_output, tf_model->model->filter_ctx);
+ tf_model->model->detect_post_proc(task->in_frame, outputs, task->nb_output, tf_model->model->filter_ctx);
break;
default:
av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n");
@@ -1123,34 +1124,6 @@ DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *
{
TFModel *tf_model = model->model;
TFContext *ctx = &tf_model->ctx;
- TaskItem task;
- TFRequestItem *request;
-
- if (ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params) != 0) {
- return DNN_ERROR;
- }
-
- if (ff_dnn_fill_task(&task, exec_params, tf_model, 0, 1) != DNN_SUCCESS) {
- return DNN_ERROR;
- }
-
- if (extract_inference_from_task(&task, tf_model->inference_queue) != DNN_SUCCESS) {
- av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
- return DNN_ERROR;
- }
-
- request = ff_safe_queue_pop_front(tf_model->request_queue);
- if (!request) {
- av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
- return DNN_ERROR;
- }
-
- return execute_model_tf(request, tf_model->inference_queue);
-}
-
-DNNReturnType ff_dnn_execute_model_async_tf(const DNNModel *model, DNNExecBaseParams *exec_params) {
- TFModel *tf_model = model->model;
- TFContext *ctx = &tf_model->ctx;
TaskItem *task;
TFRequestItem *request;
@@ -1164,7 +1137,7 @@ DNNReturnType ff_dnn_execute_model_async_tf(const DNNModel *model, DNNExecBasePa
return DNN_ERROR;
}
- if (ff_dnn_fill_task(task, exec_params, tf_model, 1, 1) != DNN_SUCCESS) {
+ if (ff_dnn_fill_task(task, exec_params, tf_model, ctx->options.async, 1) != DNN_SUCCESS) {
av_freep(&task);
return DNN_ERROR;
}
@@ -1188,10 +1161,10 @@ DNNReturnType ff_dnn_execute_model_async_tf(const DNNModel *model, DNNExecBasePa
return execute_model_tf(request, tf_model->inference_queue);
}
-DNNAsyncStatusType ff_dnn_get_async_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)
+DNNAsyncStatusType ff_dnn_get_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)
{
TFModel *tf_model = model->model;
- return ff_dnn_get_async_result_common(tf_model->task_queue, in, out);
+ return ff_dnn_get_result_common(tf_model->task_queue, in, out);
}
DNNReturnType ff_dnn_flush_tf(const DNNModel *model)
diff --git a/libavfilter/dnn/dnn_backend_tf.h b/libavfilter/dnn/dnn_backend_tf.h
index aec0fc2011..f14ea8c47a 100644
--- a/libavfilter/dnn/dnn_backend_tf.h
+++ b/libavfilter/dnn/dnn_backend_tf.h
@@ -32,8 +32,7 @@
DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params);
-DNNReturnType ff_dnn_execute_model_async_tf(const DNNModel *model, DNNExecBaseParams *exec_params);
-DNNAsyncStatusType ff_dnn_get_async_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out);
+DNNAsyncStatusType ff_dnn_get_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out);
DNNReturnType ff_dnn_flush_tf(const DNNModel *model);
void ff_dnn_free_model_tf(DNNModel **model);
diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c
index 81af934dd5..554a36b0dc 100644
--- a/libavfilter/dnn/dnn_interface.c
+++ b/libavfilter/dnn/dnn_interface.c
@@ -42,14 +42,15 @@ DNNModule *ff_get_dnn_module(DNNBackendType backend_type)
case DNN_NATIVE:
dnn_module->load_model = &ff_dnn_load_model_native;
dnn_module->execute_model = &ff_dnn_execute_model_native;
+ dnn_module->get_result = &ff_dnn_get_result_native;
+ dnn_module->flush = &ff_dnn_flush_native;
dnn_module->free_model = &ff_dnn_free_model_native;
break;
case DNN_TF:
#if (CONFIG_LIBTENSORFLOW == 1)
dnn_module->load_model = &ff_dnn_load_model_tf;
dnn_module->execute_model = &ff_dnn_execute_model_tf;
- dnn_module->execute_model_async = &ff_dnn_execute_model_async_tf;
- dnn_module->get_async_result = &ff_dnn_get_async_result_tf;
+ dnn_module->get_result = &ff_dnn_get_result_tf;
dnn_module->flush = &ff_dnn_flush_tf;
dnn_module->free_model = &ff_dnn_free_model_tf;
#else
@@ -61,8 +62,7 @@ DNNModule *ff_get_dnn_module(DNNBackendType backend_type)
#if (CONFIG_LIBOPENVINO == 1)
dnn_module->load_model = &ff_dnn_load_model_ov;
dnn_module->execute_model = &ff_dnn_execute_model_ov;
- dnn_module->execute_model_async = &ff_dnn_execute_model_async_ov;
- dnn_module->get_async_result = &ff_dnn_get_async_result_ov;
+ dnn_module->get_result = &ff_dnn_get_result_ov;
dnn_module->flush = &ff_dnn_flush_ov;
dnn_module->free_model = &ff_dnn_free_model_ov;
#else
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-10 09:51:04 +0000