avc: Add a function for converting mp4 style extradata to annex b

Make movenc use this function instead of the current custom
conversion function.

Signed-off-by: Martin Storsjö <martin@martin.st>

3 files changed, 42 insertions, 24 deletions

diff --git a/libavformat/avc.c b/libavformat/avc.c
index b0c511e7b5..2fd5ac807b 100644
--- a/libavformat/avc.c
+++ b/libavformat/avc.c
@@ -160,3 +160,34 @@ int ff_isom_write_avcc(AVIOContext *pb, const uint8_t *data, int len)
     }
     return 0;
 }
+
+int ff_avc_write_annexb_extradata(const uint8_t *in, uint8_t **buf, int *size)
+{
+    uint16_t sps_size, pps_size;
+    uint8_t *out;
+    int out_size;
+
+    *buf = NULL;
+    if (*size >= 4 && (AV_RB32(in) == 0x00000001 || AV_RB24(in) == 0x000001))
+        return 0;
+    if (*size < 11 || in[0] != 1)
+        return AVERROR_INVALIDDATA;
+
+    sps_size = AV_RB16(&in[6]);
+    if (11 + sps_size > *size)
+        return AVERROR_INVALIDDATA;
+    pps_size = AV_RB16(&in[9 + sps_size]);
+    if (11 + sps_size + pps_size > *size)
+        return AVERROR_INVALIDDATA;
+    out_size = 8 + sps_size + pps_size;
+    out = av_mallocz(out_size);
+    if (!out)
+        return AVERROR(ENOMEM);
+    AV_WB32(&out[0], 0x00000001);
+    memcpy(out + 4, &in[8], sps_size);
+    AV_WB32(&out[4 + sps_size], 0x00000001);
+    memcpy(out + 8 + sps_size, &in[11 + sps_size], pps_size);
+    *buf = out;
+    *size = out_size;
+    return 0;
+}
diff --git a/libavformat/avc.h b/libavformat/avc.h
index 56122125a9..579756ec0f 100644
--- a/libavformat/avc.h
+++ b/libavformat/avc.h
@@ -29,5 +29,6 @@ int ff_avc_parse_nal_units(AVIOContext *s, const uint8_t *buf, int size);
 int ff_avc_parse_nal_units_buf(const uint8_t *buf_in, uint8_t **buf, int *size);
 int ff_isom_write_avcc(AVIOContext *pb, const uint8_t *data, int len);
 const uint8_t *ff_avc_find_startcode(const uint8_t *p, const uint8_t *end);
+int ff_avc_write_annexb_extradata(const uint8_t *in, uint8_t **buf, int *size);
 
 #endif /* AVFORMAT_AVC_H */
diff --git a/libavformat/movenc.c b/libavformat/movenc.c
index 1209b7b114..a832c1d102 100644
--- a/libavformat/movenc.c
+++ b/libavformat/movenc.c
@@ -2096,26 +2096,6 @@ static void param_write_hex(AVIOContext *pb, const char *name, const uint8_t *va
     avio_printf(pb, "<param name=\"%s\" value=\"%s\" valuetype=\"data\"/>\n", name, buf);
 }
 
-static void write_h264_extradata(AVIOContext *pb, AVCodecContext *enc)
-{
-    uint16_t sps_size, pps_size, len;
-    char buf[150];
-    sps_size = AV_RB16(&enc->extradata[6]);
-    if (11 + sps_size > enc->extradata_size)
-        return;
-    pps_size = AV_RB16(&enc->extradata[9 + sps_size]);
-    if (11 + sps_size + pps_size > enc->extradata_size)
-        return;
-    len = FFMIN(sizeof(buf)/2 - 1, sps_size);
-    ff_data_to_hex(buf, &enc->extradata[8], len, 0);
-    buf[2*len] = '\0';
-    avio_printf(pb, "<param name=\"CodecPrivateData\" value=\"00000001%s", buf);
-    len = FFMIN(sizeof(buf)/2 - 1, pps_size);
-    ff_data_to_hex(buf, &enc->extradata[11 + sps_size], len, 0);
-    buf[2*len] = '\0';
-    avio_printf(pb, "00000001%s\" valuetype=\"data\"/>\n", buf);
-}
-
 static int mov_write_isml_manifest(AVIOContext *pb, MOVMuxContext *mov)
 {
     int64_t pos = avio_tell(pb);
@@ -2157,10 +2137,16 @@ static int mov_write_isml_manifest(AVIOContext *pb, MOVMuxContext *mov)
         param_write_int(pb, "systemBitrate", track->enc->bit_rate);
         param_write_int(pb, "trackID", track_id);
         if (track->enc->codec_type == AVMEDIA_TYPE_VIDEO) {
-            if (track->enc->codec_id == CODEC_ID_H264 &&
-                track->enc->extradata_size >= 11 &&
-                track->enc->extradata[0] == 1) {
-                write_h264_extradata(pb, track->enc);
+            if (track->enc->codec_id == CODEC_ID_H264) {
+                uint8_t *ptr;
+                int size = track->enc->extradata_size;
+                if (!ff_avc_write_annexb_extradata(track->enc->extradata, &ptr,
+                                                   &size)) {
+                    param_write_hex(pb, "CodecPrivateData",
+                                    ptr ? ptr : track->enc->extradata,
+                                    size);
+                    av_free(ptr);
+                }
             } else {
                 param_write_hex(pb, "CodecPrivateData", track->enc->extradata,
                                 track->enc->extradata_size);