Implement Realmedia/RTSP-compatible SETUP command. This includes calculation

of the "RealChallenge2" response, which is some sort of authentication. See
discussion in "Realmedia patch" thread on ffmpeg-devel.

Originally committed as revision 15170 to svn://svn.ffmpeg.org/ffmpeg/trunk

4 files changed, 143 insertions, 7 deletions

diff --git a/libavformat/Makefile b/libavformat/Makefile
index 565b2972d3..ad6b323f6b 100644
--- a/libavformat/Makefile
+++ b/libavformat/Makefile
@@ -159,7 +159,7 @@ OBJS-$(CONFIG_RTP_MUXER)                 += rtp.o         \
                                             rtp_aac.o     \
                                             rtpenc_h264.o \
                                             avc.o
-OBJS-$(CONFIG_RTSP_DEMUXER)              += rtsp.o
+OBJS-$(CONFIG_RTSP_DEMUXER)              += rdt.o rtsp.o
 OBJS-$(CONFIG_SDP_DEMUXER)               += rtsp.o rtp.o rtpdec.o rtp_h264.o
 OBJS-$(CONFIG_SEGAFILM_DEMUXER)          += segafilm.o
 OBJS-$(CONFIG_SHORTEN_DEMUXER)           += raw.o
diff --git a/libavformat/rdt.c b/libavformat/rdt.c
new file mode 100644
index 0000000000..1d0e4559d0
--- /dev/null
+++ b/libavformat/rdt.c
@@ -0,0 +1,73 @@
+/*
+ * Realmedia RTSP protocol (RDT) support.
+ * Copyright (c) 2007 Ronald S. Bultje
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file rdt.c
+ * @brief Realmedia RTSP protocol (RDT) support
+ * @author Ronald S. Bultje <rbultje@ronald.bitfreak.net>
+ */
+
+#include "avformat.h"
+#include "libavutil/avstring.h"
+#include "rdt.h"
+#include "libavutil/base64.h"
+#include "libavutil/md5.h"
+#include "rm.h"
+#include "internal.h"
+
+void
+ff_rdt_calc_response_and_checksum(char response[41], char chksum[9],
+                                  const char *challenge)
+{
+    int ch_len = strlen (challenge), i;
+    unsigned char zres[16],
+        buf[64] = { 0xa1, 0xe9, 0x14, 0x9d, 0x0e, 0x6b, 0x3b, 0x59 };
+#define XOR_TABLE_SIZE 37
+    const unsigned char xor_table[XOR_TABLE_SIZE] = {
+        0x05, 0x18, 0x74, 0xd0, 0x0d, 0x09, 0x02, 0x53,
+        0xc0, 0x01, 0x05, 0x05, 0x67, 0x03, 0x19, 0x70,
+        0x08, 0x27, 0x66, 0x10, 0x10, 0x72, 0x08, 0x09,
+        0x63, 0x11, 0x03, 0x71, 0x08, 0x08, 0x70, 0x02,
+        0x10, 0x57, 0x05, 0x18, 0x54 };
+
+    /* some (length) checks */
+    if (ch_len == 40) /* what a hack... */
+        ch_len = 32;
+    else if (ch_len > 56)
+        ch_len = 56;
+    memcpy(buf + 8, challenge, ch_len);
+
+    /* xor challenge bytewise with xor_table */
+    for (i = 0; i < XOR_TABLE_SIZE; i++)
+        buf[8 + i] ^= xor_table[i];
+
+    av_md5_sum(zres, buf, 64);
+    ff_data_to_hex(response, zres, 16);
+    for (i=0;i<32;i++) response[i] = tolower(response[i]);
+
+    /* add tail */
+    strcpy (response + 32, "01d0a8e3");
+
+    /* calculate checksum */
+    for (i = 0; i < 8; i++)
+        chksum[i] = response[i * 4];
+    chksum[8] = 0;
+}
diff --git a/libavformat/rdt.h b/libavformat/rdt.h
new file mode 100644
index 0000000000..f7aad6dfc9
--- /dev/null
+++ b/libavformat/rdt.h
@@ -0,0 +1,40 @@
+/*
+ * Realmedia RTSP (RDT) definitions
+ * Copyright (c) 2007 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVFORMAT_RDT_H
+#define AVFORMAT_RDT_H
+
+/**
+ * Calculate the response (RealChallenge2 in the RTSP header) to the
+ * challenge (RealChallenge1 in the RTSP header from the Real/Helix
+ * server), which is used as some sort of client validation.
+ *
+ * @param response pointer to response buffer, it should be at least 41 bytes
+ *                 (40 data + 1 zero) bytes long.
+ * @param chksum pointer to buffer containing a checksum of the response,
+ *               it should be at least 9 (8 data + 1 zero) bytes long.
+ * @param challenge pointer to the RealChallenge1 value provided by the
+ *                  server.
+ */
+void ff_rdt_calc_response_and_checksum(char response[41], char chksum[9],
+                                       const char *challenge);
+
+#endif /* AVFORMAT_RDT_H */
diff --git a/libavformat/rtsp.c b/libavformat/rtsp.c
index 33aa894428..389b1c5d4f 100644
--- a/libavformat/rtsp.c
+++ b/libavformat/rtsp.c
@@ -32,6 +32,7 @@
 #include "rtsp.h"
 
 #include "rtp_internal.h"
+#include "rdt.h"
 
 //#define DEBUG
 //#define DEBUG_RTP_TCP
@@ -870,7 +871,8 @@ static void rtsp_close_streams(RTSPState *rt)
  * @returns 0 on success, <0 on error, 1 if protocol is unavailable.
  */
 static int
-make_setup_request (AVFormatContext *s, const char *host, int port, int protocol)
+make_setup_request (AVFormatContext *s, const char *host, int port,
+                    int protocol, const char *real_challenge)
 {
     RTSPState *rt = s->priv_data;
     int j, i, err;
@@ -878,6 +880,12 @@ make_setup_request (AVFormatContext *s, const char *host, int port, int protocol
     AVStream *st;
     RTSPHeader reply1, *reply = &reply1;
     char cmd[2048];
+    const char *trans_pref;
+
+    if (rt->server_type == RTSP_SERVER_RDT)
+        trans_pref = "x-pn-tng";
+    else
+        trans_pref = "RTP/AVP";
 
     /* for each stream, make the setup request */
     /* XXX: we assume the same server is used for the control of each
@@ -918,8 +926,10 @@ make_setup_request (AVFormatContext *s, const char *host, int port, int protocol
             if (transport[0] != '\0')
                 av_strlcat(transport, ",", sizeof(transport));
             snprintf(transport + strlen(transport), sizeof(transport) - strlen(transport) - 1,
-                     "RTP/AVP/UDP;unicast;client_port=%d-%d",
-                     port, port + 1);
+                     "%s/UDP;unicast;client_port=%d",
+                     trans_pref, port);
+            if (rt->server_type == RTSP_SERVER_RTP)
+                av_strlcatf(transport, sizeof(transport), "-%d", port + 1);
         }
 
         /* RTP/TCP */
@@ -927,7 +937,7 @@ make_setup_request (AVFormatContext *s, const char *host, int port, int protocol
             if (transport[0] != '\0')
                 av_strlcat(transport, ",", sizeof(transport));
             snprintf(transport + strlen(transport), sizeof(transport) - strlen(transport) - 1,
-                     "RTP/AVP/TCP");
+                     "%s/TCP", trans_pref);
         }
 
         else if (protocol == RTSP_PROTOCOL_RTP_UDP_MULTICAST) {
@@ -935,12 +945,23 @@ make_setup_request (AVFormatContext *s, const char *host, int port, int protocol
                 av_strlcat(transport, ",", sizeof(transport));
             snprintf(transport + strlen(transport),
                      sizeof(transport) - strlen(transport) - 1,
-                     "RTP/AVP/UDP;multicast");
+                     "%s/UDP;multicast", trans_pref);
         }
+        if (rt->server_type == RTSP_SERVER_RDT)
+            av_strlcat(transport, ";mode=play", sizeof(transport));
         snprintf(cmd, sizeof(cmd),
                  "SETUP %s RTSP/1.0\r\n"
                  "Transport: %s\r\n",
                  rtsp_st->control_url, transport);
+        if (i == 0 && rt->server_type == RTSP_SERVER_RDT) {
+            char real_res[41], real_csum[9];
+            ff_rdt_calc_response_and_checksum(real_res, real_csum,
+                                              real_challenge);
+            av_strlcatf(cmd, sizeof(cmd),
+                        "If-Match: %s\r\n"
+                        "RealChallenge2: %s, sd=%s\r\n",
+                        rt->session_id, real_res, real_csum);
+        }
         rtsp_send_cmd(s, cmd, reply, NULL);
         if (reply->status_code == 461 /* Unsupported protocol */ && i == 0) {
             err = 1;
@@ -1155,7 +1176,9 @@ static int rtsp_read_header(AVFormatContext *s,
     do {
         int protocol = ff_log2_tab[protocol_mask & ~(protocol_mask - 1)];
 
-        err = make_setup_request(s, host, port, protocol);
+        err = make_setup_request(s, host, port, protocol,
+                                 rt->server_type == RTSP_SERVER_RDT ?
+                                     real_challenge : NULL);
         if (err < 0)
             goto fail;
         protocol_mask &= ~(1 << protocol);