h264 deblocking crash patch by (Loren Merritt <lorenm at u dot washington dot edu>)

Originally committed as revision 3427 to svn://svn.ffmpeg.org/ffmpeg/trunk

1 files changed, 21 insertions, 11 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 41806f27d9..576b337973 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -2321,6 +2321,8 @@ static inline void xchg_mb_border(H264Context *h, uint8_t *src_y, uint8_t *src_c
     MpegEncContext * const s = &h->s;
     int temp8, i;
     uint64_t temp64;
+    int deblock_left = (s->mb_x > 0);
+    int deblock_top  = (s->mb_y > 0);
 
     src_y  -=   linesize + 1;
     src_cb -= uvlinesize + 1;
@@ -2331,21 +2333,29 @@ t= a;\
 if(xchg)\
     a= b;\
 b= t;
-    
-    for(i=0; i<17; i++){
-        XCHG(h->left_border[i     ], src_y [i*  linesize], temp8, xchg);
+
+    if(deblock_left){
+        for(i = !deblock_top; i<17; i++){
+            XCHG(h->left_border[i     ], src_y [i*  linesize], temp8, xchg);
+        }
+    }
+
+    if(deblock_top){
+        XCHG(*(uint64_t*)(h->top_border[s->mb_x]+0), *(uint64_t*)(src_y +1), temp64, xchg);
+        XCHG(*(uint64_t*)(h->top_border[s->mb_x]+8), *(uint64_t*)(src_y +9), temp64, 1);
     }
-    
-    XCHG(*(uint64_t*)(h->top_border[s->mb_x]+0), *(uint64_t*)(src_y +1), temp64, xchg);
-    XCHG(*(uint64_t*)(h->top_border[s->mb_x]+8), *(uint64_t*)(src_y +9), temp64, 1);
 
     if(!(s->flags&CODEC_FLAG_GRAY)){
-        for(i=0; i<9; i++){
-            XCHG(h->left_border[i+17  ], src_cb[i*uvlinesize], temp8, xchg);
-            XCHG(h->left_border[i+17+9], src_cr[i*uvlinesize], temp8, xchg);
+        if(deblock_left){
+            for(i = !deblock_top; i<9; i++){
+                XCHG(h->left_border[i+17  ], src_cb[i*uvlinesize], temp8, xchg);
+                XCHG(h->left_border[i+17+9], src_cr[i*uvlinesize], temp8, xchg);
+            }
+        }
+        if(deblock_top){
+            XCHG(*(uint64_t*)(h->top_border[s->mb_x]+16), *(uint64_t*)(src_cb+1), temp64, 1);
+            XCHG(*(uint64_t*)(h->top_border[s->mb_x]+24), *(uint64_t*)(src_cr+1), temp64, 1);
         }
-        XCHG(*(uint64_t*)(h->top_border[s->mb_x]+16), *(uint64_t*)(src_cb+1), temp64, 1);
-        XCHG(*(uint64_t*)(h->top_border[s->mb_x]+24), *(uint64_t*)(src_cr+1), temp64, 1);
     }
 }