From 330236911b0d091d7bddfe9fe43ef7f5cac094aa Mon Sep 17 00:00:00 2001 From: Michael Giacomelli Date: Sat, 19 Sep 2026 16:16:45 -0400 Subject: [PATCH] flac: decode residuals that use all 32 bits The folded Rice value was unfolded with a signed shift, which is wrong once it reaches 2^31, and the unary length limit was (INT_MAX >> k) + 2, about half of what a 32-bit residual can need. Streams with very large residuals (FLAC decoder testbench file 63) were misparsed, overran the frame and lost sync at the next one. Unfold as unsigned and derive the limit from UINT_MAX, clamped to INT_MAX. The fast path is unchanged. The existing 0x80000000 error check now also works as intended, since the Golomb reader's error value maps to it. The FLAC spec forbids a residual of -2^31. Co-Authored-By: Claude Sonnet 5 Change-Id: I0bdce9db69e1b30565ecc23126923ba401b7deca --- lib/rbcodec/codecs/libffmpegFLAC/decoder.c | 3 ++- lib/rbcodec/codecs/libffmpegFLAC/golomb.h | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/rbcodec/codecs/libffmpegFLAC/decoder.c b/lib/rbcodec/codecs/libffmpegFLAC/decoder.c index dc5a98cf5b..9701a598b2 100644 --- a/lib/rbcodec/codecs/libffmpegFLAC/decoder.c +++ b/lib/rbcodec/codecs/libffmpegFLAC/decoder.c @@ -165,7 +165,8 @@ static int decode_residuals(FLACContext *s, int32_t *decoded, int pred_order) for (; i < samples; i++) *decoded++ = tmp ? get_sbits(&gb, tmp) : 0; } else { - int real_limit = tmp ? (INT_MAX >> tmp) + 2 : INT_MAX; + unsigned lim = tmp ? (UINT_MAX >> tmp) + 2 : INT_MAX; + int real_limit = lim > INT_MAX ? INT_MAX : lim; for (; i < samples; i++) { int v = get_sr_golomb_flac(&gb, tmp, real_limit, 0); if ((unsigned) v == 0x80000000){ diff --git a/lib/rbcodec/codecs/libffmpegFLAC/golomb.h b/lib/rbcodec/codecs/libffmpegFLAC/golomb.h index 68d2558e1d..b62051ff1a 100644 --- a/lib/rbcodec/codecs/libffmpegFLAC/golomb.h +++ b/lib/rbcodec/codecs/libffmpegFLAC/golomb.h @@ -85,7 +85,7 @@ static inline int get_ur_golomb_jpegls(GetBitContext *gb, int k, int limit, buf = 0; } - buf += ((int32_t)i << k); + buf += ((unsigned int)i << k); } else if (i == limit - 1) { buf = SHOW_UBITS(re, gb, esc_len); LAST_SKIP_BITS(re, gb, esc_len); @@ -103,8 +103,9 @@ static inline int get_ur_golomb_jpegls(GetBitContext *gb, int k, int limit, * read signed golomb rice code (flac). */ static inline int get_sr_golomb_flac(GetBitContext *gb, int k, int limit, int esc_len){ - int v= get_ur_golomb_jpegls(gb, k, limit, esc_len); - return (v>>1) ^ -(v&1); + /* The folded value can use all 32 bits, so unfold it as unsigned. */ + unsigned int v= get_ur_golomb_jpegls(gb, k, limit, esc_len); + return (int)((v>>1) ^ -(v&1)); } /**