samplefmt: change layout for arrays created by av_samples_alloc() and _fill_arrays()

The new layout is consistent with that of the av_image_() API, and
simplifies understanding and copy operations, it also preserves
alignment information which was lost with the previous layout.

This breaks API/ABI, but since the function was never referenced in
the code (and it isn't unlikely already used by someone) then this
should not be a problem.

3 files changed, 20 insertions, 13 deletions

diff --git a/libavutil/avutil.h b/libavutil/avutil.h
index cdd4f7131c..fd5f293859 100644
--- a/libavutil/avutil.h
+++ b/libavutil/avutil.h
@@ -40,7 +40,7 @@
 #define AV_VERSION(a, b, c) AV_VERSION_DOT(a, b, c)
 
 #define LIBAVUTIL_VERSION_MAJOR 51
-#define LIBAVUTIL_VERSION_MINOR  4
+#define LIBAVUTIL_VERSION_MINOR  5
 #define LIBAVUTIL_VERSION_MICRO  0
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
diff --git a/libavutil/samplefmt.c b/libavutil/samplefmt.c
index ea004d926a..ca669da8b5 100644
--- a/libavutil/samplefmt.c
+++ b/libavutil/samplefmt.c
@@ -76,28 +76,30 @@ int av_samples_fill_arrays(uint8_t *pointers[8], int linesizes[8],
                            uint8_t *buf, int nb_channels, int nb_samples,
                            enum AVSampleFormat sample_fmt, int planar, int align)
 {
-    int i, step_size = 0;
+    int i, linesize;
     int sample_size = av_get_bits_per_sample_fmt(sample_fmt) >> 3;
-    int channel_step = planar ? FFALIGN(nb_samples*sample_size, align) : sample_size;
 
-    if(nb_channels * (uint64_t)nb_samples * sample_size >= INT_MAX - align*(uint64_t)nb_channels)
+    if (nb_channels * (uint64_t)nb_samples * sample_size >= INT_MAX - align*(uint64_t)nb_channels)
         return AVERROR(EINVAL);
+    linesize = planar ? FFALIGN(nb_samples*sample_size,             align) :
+                        FFALIGN(nb_samples*sample_size*nb_channels, align);
 
     if (pointers) {
         pointers[0] = buf;
-        for (i = 0; i < nb_channels; i++) {
-            pointers[i] = buf + step_size;
-            step_size += channel_step;
+        for (i = 1; planar && i < nb_channels; i++) {
+            pointers[i] = pointers[i-1] + linesize;
         }
-        memset(&pointers[nb_channels], 0, (8-nb_channels) * sizeof(pointers[0]));
+        memset(&pointers[i], 0, (8-i) * sizeof(pointers[0]));
     }
 
     if (linesizes) {
-        linesizes[0] = planar ?  sample_size : nb_channels*sample_size;
-        memset(&linesizes[1], 0, (8-1) * sizeof(linesizes[0]));
+        linesizes[0] = linesize;
+        for (i = 1; planar && i < nb_channels; i++)
+            linesizes[i] = linesizes[0];
+        memset(&linesizes[i], 0, (8-i) * sizeof(linesizes[0]));
     }
 
-    return planar ? channel_step * nb_channels : FFALIGN(nb_channels*sample_size*nb_samples, align);
+    return planar ? linesize * nb_channels : linesize;
 }
 
 int av_samples_alloc(uint8_t *pointers[8], int linesizes[8],
diff --git a/libavutil/samplefmt.h b/libavutil/samplefmt.h
index 9b9c0d49a9..a091721d96 100644
--- a/libavutil/samplefmt.h
+++ b/libavutil/samplefmt.h
@@ -74,8 +74,13 @@ int av_get_bits_per_sample_fmt(enum AVSampleFormat sample_fmt);
  * format sample_fmt.
  *
  * The pointers array is filled with the pointers to the samples data:
- * data[c] points to the first sample of channel c.
- * data[c] + linesize[0] points to the second sample of channel c
+ * for planar, set the start point of each plane's data within the buffer,
+ * for packed, set the start point of the entire buffer only.
+ *
+ * The linesize array is filled with the aligned size of each samples
+ * plane, that is linesize[i] will contain the linesize of the plane i,
+ * and will be zero for all the unused planes. All linesize values are
+ * equal.
  *
  * @param pointers array to be filled with the pointer for each plane, may be NULL
  * @param linesizes array to be filled with the linesize, may be NULL