Merge commit '588b6215b4c74945994eb9636b0699028c069ed2'

* commit '588b6215b4c74945994eb9636b0699028c069ed2':
  rtmpcrypt: Do the xtea decryption in little endian mode
  xtea: Add functions for little endian mode

  Conflicts:
      libavutil/xtea.c

Merged-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>

5 files changed, 108 insertions, 15 deletions

diff --git a/doc/APIchanges b/doc/APIchanges
index edeba02bff..c454cd2bb4 100644
--- a/doc/APIchanges
+++ b/doc/APIchanges
@@ -14,6 +14,8 @@ libavutil:     2015-08-28
 
 
 API changes, most recent first:
+2015-11-xx - xxxxxxx - lavu 55.8.100 - xtea.h
+    Add av_xtea_le_init and av_xtea_le_crypt
 
 2015-11-18 - lavu 55.7.100 - mem.h
   Add av_fast_mallocz()
diff --git a/libavformat/rtmpcrypt.c b/libavformat/rtmpcrypt.c
index 2065ec6602..184bcf63fa 100644
--- a/libavformat/rtmpcrypt.c
+++ b/libavformat/rtmpcrypt.c
@@ -186,7 +186,7 @@ static void rtmpe8_sig(const uint8_t *in, uint8_t *out, int key_id)
     struct AVXTEA ctx;
 
     av_xtea_init(&ctx, rtmpe8_keys[key_id]);
-    av_xtea_crypt(&ctx, out, in, 1, NULL, 0);
+    av_xtea_le_crypt(&ctx, out, in, 1, NULL, 0);
 }
 
 static void rtmpe9_sig(const uint8_t *in, uint8_t *out, int key_id)
diff --git a/libavutil/version.h b/libavutil/version.h
index 8198e6ded8..4520c52909 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -56,7 +56,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  55
-#define LIBAVUTIL_VERSION_MINOR   7
+#define LIBAVUTIL_VERSION_MINOR   8
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
diff --git a/libavutil/xtea.c b/libavutil/xtea.c
index 0d58cba832..2139aa5520 100644
--- a/libavutil/xtea.c
+++ b/libavutil/xtea.c
@@ -47,6 +47,14 @@ void av_xtea_init(AVXTEA *ctx, const uint8_t key[16])
         ctx->key[i] = AV_RB32(key + (i << 2));
 }
 
+void av_xtea_le_init(AVXTEA *ctx, const uint8_t key[16])
+{
+    int i;
+
+    for (i = 0; i < 4; i++)
+        ctx->key[i] = AV_RL32(key + (i << 2));
+}
+
 static void xtea_crypt_ecb(AVXTEA *ctx, uint8_t *dst, const uint8_t *src,
                            int decrypt, uint8_t *iv)
 {
@@ -167,14 +175,51 @@ static void xtea_crypt_ecb(AVXTEA *ctx, uint8_t *dst, const uint8_t *src,
     AV_WB32(dst + 4, v1);
 }
 
-void av_xtea_crypt(AVXTEA *ctx, uint8_t *dst, const uint8_t *src, int count,
-                   uint8_t *iv, int decrypt)
+static void xtea_le_crypt_ecb(AVXTEA *ctx, uint8_t *dst, const uint8_t *src,
+                              int decrypt, uint8_t *iv)
+{
+    uint32_t v0, v1;
+    int i;
+
+    v0 = AV_RL32(src);
+    v1 = AV_RL32(src + 4);
+
+    if (decrypt) {
+        uint32_t delta = 0x9E3779B9, sum = delta * 32;
+
+        for (i = 0; i < 32; i++) {
+            v1 -= (((v0 << 4) ^ (v0 >> 5)) + v0) ^ (sum + ctx->key[(sum >> 11) & 3]);
+            sum -= delta;
+            v0 -= (((v1 << 4) ^ (v1 >> 5)) + v1) ^ (sum + ctx->key[sum & 3]);
+        }
+        if (iv) {
+            v0 ^= AV_RL32(iv);
+            v1 ^= AV_RL32(iv + 4);
+            memcpy(iv, src, 8);
+        }
+    } else {
+        uint32_t sum = 0, delta = 0x9E3779B9;
+
+        for (i = 0; i < 32; i++) {
+            v0 += (((v1 << 4) ^ (v1 >> 5)) + v1) ^ (sum + ctx->key[sum & 3]);
+            sum += delta;
+            v1 += (((v0 << 4) ^ (v0 >> 5)) + v0) ^ (sum + ctx->key[(sum >> 11) & 3]);
+        }
+    }
+
+    AV_WL32(dst, v0);
+    AV_WL32(dst + 4, v1);
+}
+
+static void xtea_crypt(AVXTEA *ctx, uint8_t *dst, const uint8_t *src, int count,
+                       uint8_t *iv, int decrypt,
+                       void (*crypt)(AVXTEA *, uint8_t *, const uint8_t *, int, uint8_t *))
 {
     int i;
 
     if (decrypt) {
         while (count--) {
-            xtea_crypt_ecb(ctx, dst, src, decrypt, iv);
+            crypt(ctx, dst, src, decrypt, iv);
 
             src   += 8;
             dst   += 8;
@@ -184,10 +229,10 @@ void av_xtea_crypt(AVXTEA *ctx, uint8_t *dst, const uint8_t *src, int count,
             if (iv) {
                 for (i = 0; i < 8; i++)
                     dst[i] = src[i] ^ iv[i];
-                xtea_crypt_ecb(ctx, dst, dst, decrypt, NULL);
+                crypt(ctx, dst, dst, decrypt, NULL);
                 memcpy(iv, dst, 8);
             } else {
-                xtea_crypt_ecb(ctx, dst, src, decrypt, NULL);
+                crypt(ctx, dst, src, decrypt, NULL);
             }
             src   += 8;
             dst   += 8;
@@ -195,6 +240,18 @@ void av_xtea_crypt(AVXTEA *ctx, uint8_t *dst, const uint8_t *src, int count,
     }
 }
 
+void av_xtea_crypt(AVXTEA *ctx, uint8_t *dst, const uint8_t *src, int count,
+                   uint8_t *iv, int decrypt)
+{
+    xtea_crypt(ctx, dst, src, count, iv, decrypt, xtea_crypt_ecb);
+}
+
+void av_xtea_le_crypt(AVXTEA *ctx, uint8_t *dst, const uint8_t *src, int count,
+                      uint8_t *iv, int decrypt)
+{
+    xtea_crypt(ctx, dst, src, count, iv, decrypt, xtea_le_crypt_ecb);
+}
+
 #ifdef TEST
 #include <stdio.h>
 
@@ -235,9 +292,10 @@ static const uint8_t xtea_test_ct[XTEA_NUM_TESTS][8] = {
 
 static void test_xtea(AVXTEA *ctx, uint8_t *dst, const uint8_t *src,
                       const uint8_t *ref, int len, uint8_t *iv, int dir,
-                      const char *test)
+                      const char *test,
+                      void (*crypt)(AVXTEA *, uint8_t *, const uint8_t *, int, uint8_t *, int))
 {
-    av_xtea_crypt(ctx, dst, src, len, iv, dir);
+    crypt(ctx, dst, src, len, iv, dir);
     if (memcmp(dst, ref, 8*len)) {
         int i;
         printf("%s failed\ngot      ", test);
@@ -254,8 +312,8 @@ static void test_xtea(AVXTEA *ctx, uint8_t *dst, const uint8_t *src,
 int main(void)
 {
     AVXTEA ctx;
-    uint8_t buf[8], iv[8];
-    int i;
+    uint8_t buf[16], iv[8];
+    int i, j;
     static const uint8_t src[32] = "HelloWorldHelloWorldHelloWorld";
     uint8_t ct[32];
     uint8_t pl[32];
@@ -263,8 +321,18 @@ int main(void)
     for (i = 0; i < XTEA_NUM_TESTS; i++) {
         av_xtea_init(&ctx, xtea_test_key[i]);
 
-        test_xtea(&ctx, buf, xtea_test_pt[i], xtea_test_ct[i], 1, NULL, 0, "encryption");
-        test_xtea(&ctx, buf, xtea_test_ct[i], xtea_test_pt[i], 1, NULL, 1, "decryption");
+        test_xtea(&ctx, buf, xtea_test_pt[i], xtea_test_ct[i], 1, NULL, 0, "encryption", av_xtea_crypt);
+        test_xtea(&ctx, buf, xtea_test_ct[i], xtea_test_pt[i], 1, NULL, 1, "decryption", av_xtea_crypt);
+
+        for (j = 0; j < 4; j++)
+            AV_WL32(&buf[4*j], AV_RB32(&xtea_test_key[i][4*j]));
+        av_xtea_le_init(&ctx, buf);
+        for (j = 0; j < 2; j++) {
+            AV_WL32(&ct[4*j], AV_RB32(&xtea_test_ct[i][4*j]));
+            AV_WL32(&pl[4*j], AV_RB32(&xtea_test_pt[i][4*j]));
+        }
+        test_xtea(&ctx, buf, pl, ct, 1, NULL, 0, "encryption", av_xtea_le_crypt);
+        test_xtea(&ctx, buf, ct, pl, 1, NULL, 1, "decryption", av_xtea_le_crypt);
 
         /* encrypt */
         memcpy(iv, "HALLO123", 8);
@@ -272,10 +340,10 @@ int main(void)
 
         /* decrypt into pl */
         memcpy(iv, "HALLO123", 8);
-        test_xtea(&ctx, pl, ct, src, 4, iv, 1, "CBC decryption");
+        test_xtea(&ctx, pl, ct, src, 4, iv, 1, "CBC decryption", av_xtea_crypt);
 
         memcpy(iv, "HALLO123", 8);
-        test_xtea(&ctx, ct, ct, src, 4, iv, 1, "CBC inplace decryption");
+        test_xtea(&ctx, ct, ct, src, 4, iv, 1, "CBC inplace decryption", av_xtea_crypt);
     }
 
     printf("Test encryption/decryption success.\n");
diff --git a/libavutil/xtea.h b/libavutil/xtea.h
index 06f5a2f470..735427c109 100644
--- a/libavutil/xtea.h
+++ b/libavutil/xtea.h
@@ -51,6 +51,15 @@ AVXTEA *av_xtea_alloc(void);
 void av_xtea_init(struct AVXTEA *ctx, const uint8_t key[16]);
 
 /**
+ * Initialize an AVXTEA context.
+ *
+ * @param ctx an AVXTEA context
+ * @param key a key of 16 bytes used for encryption/decryption,
+ *            interpreted as little endian 32 bit numbers
+ */
+void av_xtea_le_init(struct AVXTEA *ctx, const uint8_t key[16]);
+
+/**
  * Encrypt or decrypt a buffer using a previously initialized context,
  * in big endian format.
  *
@@ -65,6 +74,20 @@ void av_xtea_crypt(struct AVXTEA *ctx, uint8_t *dst, const uint8_t *src,
                    int count, uint8_t *iv, int decrypt);
 
 /**
+ * Encrypt or decrypt a buffer using a previously initialized context,
+ * in little endian format.
+ *
+ * @param ctx an AVXTEA context
+ * @param dst destination array, can be equal to src
+ * @param src source array, can be equal to dst
+ * @param count number of 8 byte blocks
+ * @param iv initialization vector for CBC mode, if NULL then ECB will be used
+ * @param decrypt 0 for encryption, 1 for decryption
+ */
+void av_xtea_le_crypt(struct AVXTEA *ctx, uint8_t *dst, const uint8_t *src,
+                      int count, uint8_t *iv, int decrypt);
+
+/**
  * @}
  */