h264: skip start code search if the size of the nal unit is known

Start code emulation prevention is only required in Annex B bytestream
packed NAL units. For other coding formats the size is already known.
Looking for a start code prefix can result in false positives like in
http://streams.videolan.org/streams/mp4/Mr_MrsSmith-h264_aac.mp4
which has a false positive in the SPS.

3 files changed, 14 insertions, 4 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 77acd7168f..5d4ce90cae 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -137,7 +137,10 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode){
     return mode;
 }
 
-const uint8_t *ff_h264_decode_nal(H264Context *h, const uint8_t *src, int *dst_length, int *consumed, int length){
+const uint8_t *ff_h264_decode_nal(H264Context *h, const uint8_t *src,
+                                  int *dst_length, int *consumed, int length,
+                                  int nalsize_known)
+{
     int i, si, di;
     uint8_t *dst;
     int bufidx;
@@ -148,6 +151,9 @@ const uint8_t *ff_h264_decode_nal(H264Context *h, const uint8_t *src, int *dst_l
 
     src++; length--;
 
+    if (nalsize_known) {
+        i = length;
+    } else
 #if HAVE_FAST_UNALIGNED
 # if HAVE_FAST_64BIT
 #   define RS 7
@@ -3789,7 +3795,8 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){
 
         hx = h->thread_context[context_count];
 
-        ptr= ff_h264_decode_nal(hx, buf + buf_index, &dst_length, &consumed, next_avc - buf_index);
+        ptr= ff_h264_decode_nal(hx, buf + buf_index, &dst_length, &consumed,
+                                next_avc - buf_index, !!nalsize);
         if (ptr==NULL || dst_length < 0){
             return -1;
         }
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 50255389fa..24da4f5eac 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -610,9 +610,12 @@ int ff_h264_decode_picture_parameter_set(H264Context *h, int bit_length);
  * @param consumed is the number of bytes used as input
  * @param length is the length of the array
  * @param dst_length is the number of decoded bytes FIXME here or a decode rbsp tailing?
+ * @param nalsize_known skip start code search if the size of the nalu is known
  * @return decoded bytes, might be src+1 if no escapes
  */
-const uint8_t *ff_h264_decode_nal(H264Context *h, const uint8_t *src, int *dst_length, int *consumed, int length);
+const uint8_t *ff_h264_decode_nal(H264Context *h, const uint8_t *src,
+                                  int *dst_length, int *consumed, int length,
+                                  int nalsize_known);
 
 /**
  * Free any data that may have been allocated in the H264 context like SPS, PPS etc.
diff --git a/libavcodec/h264_parser.c b/libavcodec/h264_parser.c
index 826c17a0f1..1967eface3 100644
--- a/libavcodec/h264_parser.c
+++ b/libavcodec/h264_parser.c
@@ -144,7 +144,7 @@ static inline int parse_nal_units(AVCodecParserContext *s,
                 src_length = 20;
             break;
         }
-        ptr= ff_h264_decode_nal(h, buf, &dst_length, &consumed, src_length);
+        ptr= ff_h264_decode_nal(h, buf, &dst_length, &consumed, src_length, 0);
         if (ptr==NULL || dst_length < 0)
             break;