frontends/va: Fix parsing leb128 when using more than 4 bytes

Bit shift would go over 32 bits. Also add assert for maximum
value as allowed by spec.

Fixes coverity issue 1469252 Bad bit shift operation

Fixes: 5edbecb856 ("frontends/va: adding va av1 encoding functions")
Acked-by: Leo Liu <leo.liu@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31622>
(cherry picked from commit 2cb3c2e8d5)
This commit is contained in:
David Rosca 2024-10-14 10:44:28 +02:00 committed by Eric Engestrom
parent e9884d1624
commit 2f2b79a495
2 changed files with 4 additions and 3 deletions

View file

@ -284,7 +284,7 @@
"description": "frontends/va: Fix parsing leb128 when using more than 4 bytes",
"nominated": true,
"nomination_type": 1,
"resolution": 0,
"resolution": 1,
"main_sha": null,
"because_sha": "5edbecb8569d88e7faa28ca7a56eb5e1672a2dd0",
"notes": null

View file

@ -84,17 +84,18 @@ static unsigned av1_uvlc(struct vl_vlc *vlc)
static unsigned av1_uleb128(struct vl_vlc *vlc)
{
unsigned value = 0;
uint64_t value = 0;
unsigned leb128Bytes = 0;
unsigned i;
for (i = 0; i < 8; ++i) {
leb128Bytes = av1_f(vlc, 8);
value |= ((leb128Bytes & 0x7f) << (i * 7));
value |= ((uint64_t)(leb128Bytes & 0x7f) << (i * 7));
if (!(leb128Bytes & 0x80))
break;
}
assert(value <= UINT32_MAX);
return value;
}