avformat/assdec: UTF-16 support

Use the UTF-16 BOM to detect UTF-16 encoding. Convert the file contents
to UTF-8 on the fly using FFTextReader, which acts as converting wrapper
around AVIOContext. It also can work on a static buffer, needed for
format probing. The FFTextReader wrapper now also takes care of skipping
the UTF-8 BOM.

Fix Ticket #3496.

1 files changed, 63 insertions, 0 deletions

diff --git a/libavformat/subtitles.c b/libavformat/subtitles.c
index fce2bf190b..cebd453965 100644
--- a/libavformat/subtitles.c
+++ b/libavformat/subtitles.c
@@ -20,9 +20,72 @@
 
 #include "avformat.h"
 #include "subtitles.h"
+#include "avio_internal.h"
 #include "libavutil/avassert.h"
 #include "libavutil/avstring.h"
 
+void ff_text_init_avio(FFTextReader *r, AVIOContext *pb)
+{
+    int i;
+    r->pb = pb;
+    r->buf_pos = r->buf_len = 0;
+    r->type = FF_UTF_8;
+    for (i = 0; i < 2; i++)
+        r->buf[r->buf_len++] = avio_r8(r->pb);
+    if (strncmp("\xFF\xFE", r->buf, 2) == 0) {
+        r->type = FF_UTF16LE;
+        r->buf_pos += 2;
+    } else if (strncmp("\xFE\xFF", r->buf, 2) == 0) {
+        r->type = FF_UTF16BE;
+        r->buf_pos += 2;
+    } else {
+        r->buf[r->buf_len++] = avio_r8(r->pb);
+        if (strncmp("\xEF\xBB\xBF", r->buf, 3) == 0) {
+            // UTF8
+            r->buf_pos += 3;
+        }
+    }
+}
+
+void ff_text_init_buf(FFTextReader *r, void *buf, size_t size)
+{
+    memset(&r->buf_pb, 0, sizeof(r->buf_pb));
+    ffio_init_context(&r->buf_pb, buf, size, 0, NULL, NULL, NULL, NULL);
+    ff_text_init_avio(r, &r->buf_pb);
+}
+
+int64_t ff_text_pos(FFTextReader *r)
+{
+    return avio_tell(r->pb) - r->buf_len + r->buf_pos;
+}
+
+int ff_text_r8(FFTextReader *r)
+{
+    uint32_t val;
+    uint8_t tmp;
+    if (r->buf_pos < r->buf_len)
+        return r->buf[r->buf_pos++];
+    if (r->type == FF_UTF16LE) {
+        GET_UTF16(val, avio_rl16(r->pb), return 0;)
+    } else if (r->type == FF_UTF16BE) {
+        GET_UTF16(val, avio_rb16(r->pb), return 0;)
+    } else {
+        return avio_r8(r->pb);
+    }
+    if (!val)
+        return 0;
+    r->buf_pos = 0;
+    r->buf_len = 0;
+    PUT_UTF8(val, tmp, r->buf[r->buf_len++] = tmp;)
+    return r->buf[r->buf_pos++]; // buf_len is at least 1
+}
+
+void ff_text_read(FFTextReader *r, char *buf, size_t size)
+{
+    for ( ; size > 0; size--)
+        *buf++ = ff_text_r8(r);
+}
+
 AVPacket *ff_subtitles_queue_insert(FFDemuxSubtitlesQueue *q,
                                     const uint8_t *event, int len, int merge)
 {