fix assertion failure

Originally committed as revision 9152 to svn://svn.ffmpeg.org/ffmpeg/trunk

2 files changed, 11 insertions, 9 deletions

diff --git a/libavcodec/rangecoder.h b/libavcodec/rangecoder.h
index 68bd3b60e4..3aac8c6238 100644
--- a/libavcodec/rangecoder.h
+++ b/libavcodec/rangecoder.h
@@ -66,6 +66,13 @@ static inline void renorm_encoder(RangeCoder *c){
     }
 }
 
+static inline int get_rac_count(RangeCoder *c){
+    int x= c->bytestream - c->bytestream_start + c->outstanding_count;
+    if(c->outstanding_byte >= 0)
+        x++;
+    return 8*x - av_log2(c->range);
+}
+
 static inline void put_rac(RangeCoder *c, uint8_t * const state, int bit){
     int range1= (c->range * (*state)) >> 8;
 
diff --git a/libavcodec/snow.c b/libavcodec/snow.c
index df32d12632..fddffc1e75 100644
--- a/libavcodec/snow.c
+++ b/libavcodec/snow.c
@@ -1989,7 +1989,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
     RangeCoder pc, ic;
     uint8_t *pbbak= s->c.bytestream;
     uint8_t *pbbak_start= s->c.bytestream_start;
-    int score, score2, iscore, i_len, p_len, block_s, sum;
+    int score, score2, iscore, i_len, p_len, block_s, sum, base_bits;
     const int w= s->b_width  << s->block_max_depth;
     const int h= s->b_height << s->block_max_depth;
     const int rem_depth= s->block_max_depth - level;
@@ -2115,6 +2115,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
     //FIXME if mb_cmp != SSE then intra cant be compared currently and mb_penalty vs. lambda2
 
   //  subpel search
+    base_bits= get_rac_count(&s->c) - (s->c.bytestream - s->c.bytestream_start);
     pc= s->c;
     pc.bytestream_start=
     pc.bytestream= p_buffer; //FIXME end/start? and at the other stoo
@@ -2129,10 +2130,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
     put_symbol(&pc, &p_state[128 + 32*(mx_context + 16*!!best_ref)], mx - pmx, 1);
     put_symbol(&pc, &p_state[128 + 32*(my_context + 16*!!best_ref)], my - pmy, 1);
     p_len= pc.bytestream - pc.bytestream_start;
-    score += (s->lambda2*(p_len*8
-              + (pc.outstanding_count - s->c.outstanding_count)*8
-              + (-av_log2(pc.range)    + av_log2(s->c.range))
-             ))>>FF_LAMBDA_SHIFT;
+    score += (s->lambda2*(get_rac_count(&pc)-base_bits))>>FF_LAMBDA_SHIFT;
 
     block_s= block_w*block_w;
     sum = pix_sum(current_data[0], stride, block_w);
@@ -2158,10 +2156,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
     put_symbol(&ic, &i_state[64], cb-pcb, 1);
     put_symbol(&ic, &i_state[96], cr-pcr, 1);
     i_len= ic.bytestream - ic.bytestream_start;
-    iscore += (s->lambda2*(i_len*8
-              + (ic.outstanding_count - s->c.outstanding_count)*8
-              + (-av_log2(ic.range)    + av_log2(s->c.range))
-             ))>>FF_LAMBDA_SHIFT;
+    iscore += (s->lambda2*(get_rac_count(&ic)-base_bits))>>FF_LAMBDA_SHIFT;
 
 //    assert(score==256*256*256*64-1);
     assert(iscore < 255*255*256 + s->lambda2*10);