avformat/movenc: Support encryption of H.265 stream in AnnexB format

Steps to test:

1. Create h265 test files - mp4 and h265 AnnexB streams:
ffmpeg -f lavfi -i testsrc=duration=10:size=640x480:rate=30 -c:v hevc input_h265.mp4
ffmpeg -f lavfi -i testsrc=duration=10:size=640x480:rate=30 -c:v hevc -bsf:v hevc_mp4toannexb input_h265.h265

2. Encrypt and decrypt files. Put appropriate input file name here: input_h265.mp4 or input_h265.h265
ffmpeg -i input_h265.h265 -vcodec copy -acodec copy -encryption_scheme cenc-aes-ctr \
 -encryption_key 00000000000000000000000000000000 -encryption_kid 00000000000000000000000000000000 \
 encrypted_h265.mp4
ffplay -i encrypted_h265.mp4  -decryption_key 00000000000000000000000000000000

Signed-off-by: Vadym Bezdushnyi <vadim.bezdush@gmail.com>

1 files changed, 14 insertions, 2 deletions

diff --git a/libavformat/movenc.c b/libavformat/movenc.c
index 04f3e94158..8fdce359db 100644
--- a/libavformat/movenc.c
+++ b/libavformat/movenc.c
@@ -5657,7 +5657,15 @@ int ff_mov_write_packet(AVFormatContext *s, AVPacket *pkt)
                 return ret;
             avio_write(pb, reformatted_data, size);
         } else {
-            size = ff_hevc_annexb2mp4(pb, pkt->data, pkt->size, 0, NULL);
+            if (trk->cenc.aes_ctr) {
+                size = ff_mov_cenc_avc_parse_nal_units(&trk->cenc, pb, pkt->data, size);
+                if (size < 0) {
+                    ret = size;
+                    goto err;
+                }
+            } else {
+                size = ff_hevc_annexb2mp4(pb, pkt->data, pkt->size, 0, NULL);
+            }
         }
     } else if (par->codec_id == AV_CODEC_ID_AV1) {
         if (trk->hint_track >= 0 && trk->hint_track < mov->nb_streams) {
@@ -5699,6 +5707,9 @@ int ff_mov_write_packet(AVFormatContext *s, AVPacket *pkt)
             if (par->codec_id == AV_CODEC_ID_H264 && par->extradata_size > 4) {
                 int nal_size_length = (par->extradata[4] & 0x3) + 1;
                 ret = ff_mov_cenc_avc_write_nal_units(s, &trk->cenc, nal_size_length, pb, pkt->data, size);
+            } else if(par->codec_id == AV_CODEC_ID_HEVC && par->extradata_size > 21) {
+                int nal_size_length = (par->extradata[21] & 0x3) + 1;
+                ret = ff_mov_cenc_avc_write_nal_units(s, &trk->cenc, nal_size_length, pb, pkt->data, size);
             } else {
                 ret = ff_mov_cenc_write_packet(&trk->cenc, pb, pkt->data, size);
             }
@@ -6710,7 +6721,8 @@ static int mov_init(AVFormatContext *s)
 
         if (mov->encryption_scheme == MOV_ENC_CENC_AES_CTR) {
             ret = ff_mov_cenc_init(&track->cenc, mov->encryption_key,
-                track->par->codec_id == AV_CODEC_ID_H264, s->flags & AVFMT_FLAG_BITEXACT);
+                (track->par->codec_id == AV_CODEC_ID_H264 || track->par->codec_id == AV_CODEC_ID_HEVC),
+                s->flags & AVFMT_FLAG_BITEXACT);
             if (ret)
                 return ret;
         }