avfilter/formats: Make ff_formats_pixdesc_filter return AVFilterFormats*

Up until now, it has returned the AVFilterFormats list via
an AVFilterFormats** parameter; the actual return value was an int
that was always AVERROR(ENOMEM) on error. The AVFilterFormats**
argument was a pure output parameter which was only documented
by naming the parameter rfmts. Yet nevertheless all callers
initialized the underlying AVFilterFormats* to NULL.

This commit changes this to return a pointer to AVFilterFormats
directly. This is more in line with the API in general, as it
allows to avoid checks for intermediate values.

Reviewed-by: Nicolas George <george@nsup.org>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

14 files changed, 46 insertions, 106 deletions

diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index 1bf7d36195..59ea966424 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -452,7 +452,7 @@ AVFilterFormats *ff_all_formats(enum AVMediaType type)
     return ret;
 }
 
-int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned rej)
+AVFilterFormats *ff_formats_pixdesc_filter(unsigned want, unsigned rej)
 {
     unsigned nb_formats, fmt, flags;
     AVFilterFormats *formats = NULL;
@@ -476,18 +476,17 @@ int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned r
         }
         if (formats) {
             av_assert0(formats->nb_formats == nb_formats);
-            *rfmts = formats;
-            return 0;
+            return formats;
         }
         formats = av_mallocz(sizeof(*formats));
         if (!formats)
-            return AVERROR(ENOMEM);
+            return NULL;
         formats->nb_formats = nb_formats;
         if (nb_formats) {
             formats->formats = av_malloc_array(nb_formats, sizeof(*formats->formats));
             if (!formats->formats) {
                 av_freep(&formats);
-                return AVERROR(ENOMEM);
+                return NULL;
             }
         }
     }
diff --git a/libavfilter/formats.h b/libavfilter/formats.h
index 471cb42bc4..42fe068765 100644
--- a/libavfilter/formats.h
+++ b/libavfilter/formats.h
@@ -230,7 +230,7 @@ AVFilterFormats *ff_all_formats(enum AVMediaType type);
  * properties
  */
 av_warn_unused_result
-int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned rej);
+AVFilterFormats *ff_formats_pixdesc_filter(unsigned want, unsigned rej);
 
 //* format is software, non-planar with sub-sampling
 #define FF_PIX_FMT_FLAG_SW_FLAT_SUB (1 << 24)
diff --git a/libavfilter/vf_copy.c b/libavfilter/vf_copy.c
index 16fbe438a1..0ed61324e7 100644
--- a/libavfilter/vf_copy.c
+++ b/libavfilter/vf_copy.c
@@ -29,14 +29,7 @@
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    int ret;
-
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, AV_PIX_FMT_FLAG_HWACCEL));
 }
 
 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
diff --git a/libavfilter/vf_crop.c b/libavfilter/vf_crop.c
index 76d375cdfc..27ae1b8118 100644
--- a/libavfilter/vf_crop.c
+++ b/libavfilter/vf_crop.c
@@ -93,13 +93,9 @@ typedef struct CropContext {
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    int ret;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM | FF_PIX_FMT_FLAG_SW_FLAT_SUB;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0, AV_PIX_FMT_FLAG_BITSTREAM | FF_PIX_FMT_FLAG_SW_FLAT_SUB);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static av_cold void uninit(AVFilterContext *ctx)
diff --git a/libavfilter/vf_detelecine.c b/libavfilter/vf_detelecine.c
index 7f34a88217..e36e1a6245 100644
--- a/libavfilter/vf_detelecine.c
+++ b/libavfilter/vf_detelecine.c
@@ -124,16 +124,11 @@ static av_cold int init(AVFilterContext *ctx)
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    int ret;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
+                       AV_PIX_FMT_FLAG_PAL       |
+                       AV_PIX_FMT_FLAG_HWACCEL;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static int config_input(AVFilterLink *inlink)
diff --git a/libavfilter/vf_fieldhint.c b/libavfilter/vf_fieldhint.c
index c6c2a8eebe..c41f2ef751 100644
--- a/libavfilter/vf_fieldhint.c
+++ b/libavfilter/vf_fieldhint.c
@@ -77,16 +77,11 @@ static av_cold int init(AVFilterContext *ctx)
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    int ret;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
+                       AV_PIX_FMT_FLAG_HWACCEL   |
+                       AV_PIX_FMT_FLAG_PAL;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_HWACCEL |
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static int config_input(AVFilterLink *inlink)
diff --git a/libavfilter/vf_hwdownload.c b/libavfilter/vf_hwdownload.c
index f00f105f84..5bd63a8a8f 100644
--- a/libavfilter/vf_hwdownload.c
+++ b/libavfilter/vf_hwdownload.c
@@ -37,13 +37,12 @@ typedef struct HWDownloadContext {
 
 static int hwdownload_query_formats(AVFilterContext *avctx)
 {
-    AVFilterFormats *fmts;
     int err;
 
-    if ((err = ff_formats_pixdesc_filter(&fmts, AV_PIX_FMT_FLAG_HWACCEL, 0)) ||
-        (err = ff_formats_ref(fmts, &avctx->inputs[0]->outcfg.formats))      ||
-        (err = ff_formats_pixdesc_filter(&fmts, 0, AV_PIX_FMT_FLAG_HWACCEL)) ||
-        (err = ff_formats_ref(fmts, &avctx->outputs[0]->incfg.formats)))
+    if ((err = ff_formats_ref(ff_formats_pixdesc_filter(AV_PIX_FMT_FLAG_HWACCEL, 0),
+                              &avctx->inputs[0]->outcfg.formats))  ||
+        (err = ff_formats_ref(ff_formats_pixdesc_filter(0, AV_PIX_FMT_FLAG_HWACCEL),
+                              &avctx->outputs[0]->incfg.formats)))
         return err;
 
     return 0;
diff --git a/libavfilter/vf_il.c b/libavfilter/vf_il.c
index 8e829b2f69..38ef1ac47a 100644
--- a/libavfilter/vf_il.c
+++ b/libavfilter/vf_il.c
@@ -83,15 +83,9 @@ AVFILTER_DEFINE_CLASS(il);
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    int ret;
+    int reject_flags = AV_PIX_FMT_FLAG_PAL | AV_PIX_FMT_FLAG_HWACCEL;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static int config_input(AVFilterLink *inlink)
diff --git a/libavfilter/vf_mix.c b/libavfilter/vf_mix.c
index ce66202557..13032e466a 100644
--- a/libavfilter/vf_mix.c
+++ b/libavfilter/vf_mix.c
@@ -55,16 +55,11 @@ typedef struct MixContext {
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    int ret;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
+                       AV_PIX_FMT_FLAG_HWACCEL   |
+                       AV_PIX_FMT_FLAG_PAL;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static int parse_weights(AVFilterContext *ctx)
diff --git a/libavfilter/vf_stack.c b/libavfilter/vf_stack.c
index 3afcdfda87..338154e715 100644
--- a/libavfilter/vf_stack.c
+++ b/libavfilter/vf_stack.c
@@ -60,21 +60,16 @@ typedef struct StackContext {
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
     StackContext *s = ctx->priv;
-    int ret;
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
+                       AV_PIX_FMT_FLAG_HWACCEL   |
+                       AV_PIX_FMT_FLAG_PAL;
 
     if (s->fillcolor_enable) {
         return ff_set_common_formats(ctx, ff_draw_supported_pixel_formats(0));
     }
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_HWACCEL |
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static av_cold int init(AVFilterContext *ctx)
diff --git a/libavfilter/vf_swaprect.c b/libavfilter/vf_swaprect.c
index fff9b53dc4..e03603fe5a 100644
--- a/libavfilter/vf_swaprect.c
+++ b/libavfilter/vf_swaprect.c
@@ -57,16 +57,11 @@ AVFILTER_DEFINE_CLASS(swaprect);
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *pix_fmts = NULL;
-    int ret;
+    int reject_flags = AV_PIX_FMT_FLAG_PAL     |
+                       AV_PIX_FMT_FLAG_HWACCEL |
+                       AV_PIX_FMT_FLAG_BITSTREAM;
 
-    ret = ff_formats_pixdesc_filter(&pix_fmts, 0, AV_PIX_FMT_FLAG_PAL     |
-                                                  AV_PIX_FMT_FLAG_HWACCEL |
-                                                  AV_PIX_FMT_FLAG_BITSTREAM);
-    if (ret < 0)
-        return ret;
-
-    return ff_set_common_formats(ctx, pix_fmts);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static const char *const var_names[] = {   "w",   "h",   "a",   "n",   "t",   "pos",   "sar",   "dar",        NULL };
diff --git a/libavfilter/vf_telecine.c b/libavfilter/vf_telecine.c
index 99c6d22b7d..eaaddf734d 100644
--- a/libavfilter/vf_telecine.c
+++ b/libavfilter/vf_telecine.c
@@ -101,16 +101,11 @@ static av_cold int init(AVFilterContext *ctx)
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    int ret;
-
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    int reject_flags = AV_PIX_FMT_FLAG_BITSTREAM |
+                       AV_PIX_FMT_FLAG_HWACCEL   |
+                       AV_PIX_FMT_FLAG_PAL;
+
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static int config_input(AVFilterLink *inlink)
diff --git a/libavfilter/vf_untile.c b/libavfilter/vf_untile.c
index 2bf5146591..85e1d56d2b 100644
--- a/libavfilter/vf_untile.c
+++ b/libavfilter/vf_untile.c
@@ -62,16 +62,11 @@ static av_cold int init(AVFilterContext *ctx)
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    int ret;
-
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_HWACCEL |
-                                    AV_PIX_FMT_FLAG_BITSTREAM |
-                                    FF_PIX_FMT_FLAG_SW_FLAT_SUB);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    int reject_flags = AV_PIX_FMT_FLAG_HWACCEL   |
+                       AV_PIX_FMT_FLAG_BITSTREAM |
+                       FF_PIX_FMT_FLAG_SW_FLAT_SUB;
+
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static int config_output(AVFilterLink *outlink)
diff --git a/libavfilter/vf_weave.c b/libavfilter/vf_weave.c
index 60e37a1037..52fb4c684d 100644
--- a/libavfilter/vf_weave.c
+++ b/libavfilter/vf_weave.c
@@ -51,15 +51,9 @@ AVFILTER_DEFINE_CLASS_EXT(weave, "(double)weave", weave_options);
 
 static int query_formats(AVFilterContext *ctx)
 {
-    AVFilterFormats *formats = NULL;
-    int ret;
+    int reject_flags = AV_PIX_FMT_FLAG_PAL | AV_PIX_FMT_FLAG_HWACCEL;
 
-    ret = ff_formats_pixdesc_filter(&formats, 0,
-                                    AV_PIX_FMT_FLAG_PAL |
-                                    AV_PIX_FMT_FLAG_HWACCEL);
-    if (ret < 0)
-        return ret;
-    return ff_set_common_formats(ctx, formats);
+    return ff_set_common_formats(ctx, ff_formats_pixdesc_filter(0, reject_flags));
 }
 
 static int config_props_output(AVFilterLink *outlink)