CVE-2023-43787: Integer overflow in XCreateImage() leading to a heap overflow

When the format is `Pixmap` it calculates the size of the image data as:
    ROUNDUP((bits_per_pixel * width), image->bitmap_pad);
There is no validation on the `width` of the image, and so this
calculation exceeds the capacity of a 4-byte integer, causing an overflow.

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
This commit is contained in:
Yair Mizrahi 2023-09-07 16:15:32 -07:00 committed by Alan Coopersmith
parent b4031fc023
commit 7916869d16

View file

@ -30,6 +30,7 @@ in this Software without prior written authorization from The Open Group.
#include <X11/Xlibint.h>
#include <X11/Xutil.h>
#include <stdio.h>
#include <limits.h>
#include "ImUtil.h"
static int _XDestroyImage(XImage *);
@ -361,13 +362,22 @@ XImage *XCreateImage (
/*
* compute per line accelerator.
*/
{
if (format == ZPixmap)
if (format == ZPixmap) {
if ((INT_MAX / bits_per_pixel) < width) {
Xfree(image);
return NULL;
}
min_bytes_per_line =
ROUNDUP((bits_per_pixel * width), image->bitmap_pad);
else
ROUNDUP((bits_per_pixel * width), image->bitmap_pad);
} else {
if ((INT_MAX - offset) < width) {
Xfree(image);
return NULL;
}
min_bytes_per_line =
ROUNDUP((width + offset), image->bitmap_pad);
ROUNDUP((width + offset), image->bitmap_pad);
}
if (image_bytes_per_line == 0) {
image->bytes_per_line = min_bytes_per_line;