dnn_backend_native: Add overflow check for length calculation.

We should not silently allocate an incorrect sized buffer.
Fixes trac issue #8718.

Signed-off-by: Reimar Döffinger <Reimar.Doeffinger@gmx.de>
Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Reviewed-by: Guo, Yejun <yejun.guo@intel.com>

1 files changed, 2 insertions, 0 deletions

diff --git a/libavfilter/dnn/dnn_backend_native_layer_maximum.c b/libavfilter/dnn/dnn_backend_native_layer_maximum.c
index af16e08b95..cdddfdd87b 100644
--- a/libavfilter/dnn/dnn_backend_native_layer_maximum.c
+++ b/libavfilter/dnn/dnn_backend_native_layer_maximum.c
@@ -64,6 +64,8 @@ int dnn_execute_layer_maximum(DnnOperand *operands, const int32_t *input_operand
 
     output->data_type = input->data_type;
     output->length = calculate_operand_data_length(output);
+    if (output->length <= 0)
+        return DNN_ERROR;
     output->data = av_realloc(output->data, output->length);
     if (!output->data)
         return DNN_ERROR;