fdo-mirrors/mesa
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/mesa/mesa.git synced 2026-05-09 04:38:03 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

util: Fixed crash in HEVC encoding on 32-bit systems

Browse source 
This builds on https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25059, and extends that change to all 32-bit systems.

This fixes a crash on SteamOS with the following test case:

        unsigned char data[] = {
          0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60,
          0x00, 0x00, 0x03, 0x00, 0xb0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00,
          0x99, 0x2c, 0x0c, 0x01, 0x64, 0x7c, 0x00, 0x7c, 0xd2, 0x56, 0x01, 0x40,
          0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03,
          0x00, 0xb0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x99, 0xa0, 0x02,
          0x80, 0x80, 0x32, 0x16, 0x24, 0xbb, 0x90, 0x84, 0x48, 0x9a, 0x83, 0x03,
          0x03, 0x02, 0x00, 0xb2, 0x3e, 0x00, 0x3e, 0x69, 0x2b, 0x00, 0x5f, 0x08,
          0x04, 0x10, 0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc0, 0x62, 0x0f, 0x02,
          0x24
        };
        vlVaContext context;
        vlVaBuffer buf;

        memset(&context, 0, sizeof(context));
        memset(&buf, 0, sizeof(buf));
        context.packed_header_emulation_bytes = true;
        buf.data = data;
        buf.size = sizeof(data);
        vlVaHandleVAEncPackedHeaderDataBufferTypeHEVC(&context, &buf);

Cc: mesa-stable
Reviewed-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
(cherry picked from commit 4ed8ef74b4)

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32261>
This commit is contained in:
Sam Lantinga 2024-10-11 08:00:52 -07:00 • committed by Dylan Baker
parent b4b12c6708
commit 922a339d91
2 changed files with 1 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.pick_status.json
View file

@ -894,7 +894,7 @@
"description": "util: Fixed crash in HEVC encoding on 32-bit systems",
"nominated": true,
"nomination_type": 1,
"resolution": 0,
"resolution": 1,
"main_sha": null,
"because_sha": null,
"notes": null

6
src/util/vl_vlc.h
View file

@ -346,14 +346,8 @@ vl_vlc_search_byte(struct vl_vlc *vlc, unsigned num_bits, uint8_t value)
static inline void
vl_vlc_removebits(struct vl_vlc *vlc, unsigned pos, unsigned num_bits)
{
#if defined(_MSC_VER)
/* MSVC Compiler defines unsigned long as 4 bytes so use explicit 64 bits mask */
uint64_t lo = (vlc->buffer & (UINT64_MAX >> (pos + num_bits))) << num_bits;
uint64_t hi = (vlc->buffer & (UINT64_MAX << (64 - pos)));
#else
uint64_t lo = (vlc->buffer & (~0UL >> (pos + num_bits))) << num_bits;
uint64_t hi = (vlc->buffer & (~0UL << (64 - pos)));
#endif
vlc->buffer = lo | hi;
vlc->invalid_bits += num_bits;
}