pyxtest: add test cases for the RandR extension CVEs of the last years

Commit 541ab2ecd4 ("Xi/randr: fix handling of PropModeAppend/Prepend")
Commit 14f480010a ("randr: avoid integer truncation in length check of ProcRRChange*Property")

Assisted-by: Claude:claude-claude-opus-4-6
Part-of: <https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/2187>

test/pyxtest/meson.build

@@ -55,6 +55,7 @@ if pytest.found()
 
     # This needs to be kept in sync with the test_foo.py files in the tree
     tests_pyxtest = [
+        'test_randr.py',
         'test_xi.py',
     ]
 

test/pyxtest/proto/randr.py

@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: MIT
+#
+# RandR extension protocol request builders
+
+import struct
+from dataclasses import dataclass
+
+# RandR minor opcodes
+RRQueryVersion = 0
+RRGetScreenResources = 8
+RRChangeOutputProperty = 13
+RRGetOutputProperty = 15
+RRGetScreenResourcesCurrent = 25
+
+RR_MAJOR = 1
+RR_MINOR = 6
+
+# Property modes (same as core X11)
+PropModeReplace = 0
+PropModePrepend = 1
+PropModeAppend = 2
+
+
+@dataclass
+class QueryVersionRequest:
+    """RRQueryVersion request."""
+
+    opcode: int
+    major: int = RR_MAJOR
+    minor: int = RR_MINOR
+
+    def to_bytes(self, byte_order: str = "<") -> bytes:
+        return struct.pack(
+            f"{byte_order}BBHII",
+            self.opcode,
+            RRQueryVersion,
+            3,
+            self.major,
+            self.minor,
+        )
+
+
+@dataclass
+class GetScreenResourcesCurrentRequest:
+    """RRGetScreenResourcesCurrent request."""
+
+    opcode: int
+    window: int
+
+    def to_bytes(self, byte_order: str = "<") -> bytes:
+        return struct.pack(
+            f"{byte_order}BBHI",
+            self.opcode,
+            RRGetScreenResourcesCurrent,
+            2,
+            self.window,
+        )
+
+
+@dataclass
+class ChangeOutputPropertyRequest:
+    """RRChangeOutputProperty request."""
+
+    opcode: int
+    output: int
+    property_atom: int
+    type_atom: int
+    format: int = 32
+    mode: int = PropModeReplace
+    data: bytes = b""
+    num_items: int | None = None
+    length_override: int | None = None
+
+    def to_bytes(self, byte_order: str = "<") -> bytes:
+        num_items = (
+            self.num_items
+            if self.num_items is not None
+            else (len(self.data) // (self.format // 8) if self.data else 0)
+        )
+
+        total_bytes = 24 + len(self.data)
+        pad_len = (4 - total_bytes % 4) % 4
+        total_bytes += pad_len
+
+        length = (
+            self.length_override
+            if self.length_override is not None
+            else total_bytes // 4
+        )
+
+        header = struct.pack(
+            f"{byte_order}BBH III BB H I",
+            self.opcode,
+            RRChangeOutputProperty,
+            length,
+            self.output,
+            self.property_atom,
+            self.type_atom,
+            self.format,
+            self.mode,
+            0,  # pad
+            num_items,
+        )
+        return header + self.data + b"\x00" * pad_len
+
+
+@dataclass
+class GetOutputPropertyRequest:
+    """RRGetOutputProperty request."""
+
+    opcode: int
+    output: int
+    property_atom: int
+    type_atom: int = 0  # AnyPropertyType
+    offset: int = 0
+    length: int = 0xFFFF
+    delete: bool = False
+    pending: bool = True
+
+    def to_bytes(self, byte_order: str = "<") -> bytes:
+        return struct.pack(
+            f"{byte_order}BBH I II II BB H",
+            self.opcode,
+            RRGetOutputProperty,
+            7,  # 28 bytes = 7 words
+            self.output,
+            self.property_atom,
+            self.type_atom,
+            self.offset,
+            self.length,
+            1 if self.delete else 0,
+            1 if self.pending else 0,
+            0,  # pad
+        )

test/pyxtest/test_randr.py

@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: MIT
+#
+# Security tests for RandR extension vulnerabilities.
+
+import struct
+
+import pytest
+
+from proto import randr
+from xclient import Extension, X11Error, X11Reply
+
+
+def _get_first_output(xclient, opcode):
+    """Return the first RandR output ID, or skip if none available."""
+    req = randr.GetScreenResourcesCurrentRequest(
+        opcode=opcode,
+        window=xclient.root_window,
+    )
+    xclient.send_request(req.to_bytes())
+    resp = xclient.recv_response(timeout=5.0)
+    if not isinstance(resp, X11Reply) or len(resp.data) < 32:
+        pytest.skip("Failed to get RandR screen resources")
+
+    n_crtcs = struct.unpack_from("<H", resp.data, 16)[0]
+    n_outputs = struct.unpack_from("<H", resp.data, 18)[0]
+    if n_outputs == 0:
+        pytest.skip("No RandR outputs available")
+
+    offset = 32 + n_crtcs * 4
+    return struct.unpack_from("<I", resp.data, offset)[0]
+
+
+@pytest.fixture
+def randr_xclient(xclient):
+    """Provide an xclient with RandR initialized, returning (xclient, opcode, output_id)."""
+    ext = xclient.query_extension(Extension.RANDR)
+    if not ext:
+        pytest.skip("RANDR extension not available")
+
+    req = randr.QueryVersionRequest(opcode=ext.opcode)
+    xclient.send_request(req.to_bytes())
+    xclient.recv_response(timeout=5.0)
+
+    output_id = _get_first_output(xclient, ext.opcode)
+    return xclient, ext.opcode, output_id
+
+
+class TestRandROutputProperty:
+    """Tests for RRChangeOutputProperty vulnerabilities."""
+
+    def test_prepend_property_size_and_offset(self, xserver, randr_xclient):
+        """
+        CVE-2023-5367 / ZDI-CAN-22153: Incorrect size and offset
+        calculation when prepending to RandR output properties.
+
+        Two bugs in RRChangeOutputProperty (copy-pasted from the XI code):
+        1. new_value.size was set to ``len`` instead of ``total_len``
+           (new + existing), so the property lost the old data's size.
+        2. The old_data offset for PropModePrepend used
+           ``prop_value->size`` instead of ``len``, placing old data at
+           the wrong position and writing out of bounds.
+
+        This test sets a property, prepends to it, and reads back the
+        result.  On a fixed server the property contains all values in
+        the correct order.
+
+        Fixed in commit 541ab2ecd41d ("Xi/randr: fix handling of
+        PropModeAppend/Prepend").
+        """
+        xclient, opcode, output_id = randr_xclient
+
+        prop_atom = xclient.intern_atom("_TEST_RR_PREPEND")
+        type_atom = xclient.intern_atom("INTEGER")
+
+        # Step 1: Set initial property with values [10, 20, 30]
+        initial_data = struct.pack("<III", 10, 20, 30)
+        req = randr.ChangeOutputPropertyRequest(
+            opcode=opcode,
+            output=output_id,
+            property_atom=prop_atom,
+            type_atom=type_atom,
+            format=32,
+            mode=randr.PropModeReplace,
+            data=initial_data,
+        )
+        xclient.send_request(req.to_bytes())
+        xclient.flush_responses(timeout=0.5)
+
+        # Step 2: Prepend values [1, 2]
+        prepend_data = struct.pack("<II", 1, 2)
+        req = randr.ChangeOutputPropertyRequest(
+            opcode=opcode,
+            output=output_id,
+            property_atom=prop_atom,
+            type_atom=type_atom,
+            format=32,
+            mode=randr.PropModePrepend,
+            data=prepend_data,
+        )
+        xclient.send_request(req.to_bytes())
+        xclient.flush_responses(timeout=0.5)
+
+        assert xserver.is_alive, (
+            "Server crashed - OOB write in RRChangeOutputProperty prepend (CVE-2023-5367)"
+        )
+
+        # Step 3: Read back and verify
+        req = randr.GetOutputPropertyRequest(
+            opcode=opcode,
+            output=output_id,
+            property_atom=prop_atom,
+            type_atom=type_atom,
+        )
+        xclient.send_request(req.to_bytes())
+        resp = xclient.recv_response(timeout=2.0)
+
+        assert isinstance(resp, X11Reply), f"Expected a reply, got {resp}"
+        num_items = struct.unpack_from("<I", resp.data, 16)[0]
+        assert num_items == 5, (
+            f"Expected 5 items (2 prepended + 3 original), got {num_items}"
+        )
+
+        values = struct.unpack_from(f"<{num_items}I", resp.data, 32)
+        assert values == (1, 2, 10, 20, 30), (
+            f"Expected (1, 2, 10, 20, 30), got {values}"
+        )
+
+    def test_change_output_property_num_items_overflow(self, xserver, randr_xclient):
+        """
+        CVE-2023-6478 / ZDI-CAN-22561: Integer truncation in
+        ProcRRChangeOutputProperty length check.
+
+        ``totalSize = nUnits * sizeInBytes`` was computed as a 32-bit int.
+        With format=32 and nUnits=0x40000000, the multiplication overflows
+        to 0, passing the REQUEST_FIXED_SIZE check.
+
+        The fix changed totalSize from ``int`` to ``uint64_t``.
+
+        Fixed in commit 14f480010a93 ("randr: avoid integer truncation in
+        length check of ProcRRChange*Property").
+        """
+        xclient, opcode, output_id = randr_xclient
+
+        prop_atom = xclient.intern_atom("_TEST_RR_OVERFLOW")
+        type_atom = xclient.intern_atom("INTEGER")
+
+        req = randr.ChangeOutputPropertyRequest(
+            opcode=opcode,
+            output=output_id,
+            property_atom=prop_atom,
+            type_atom=type_atom,
+            format=32,
+            mode=randr.PropModeReplace,
+            num_items=0x40000000,
+            data=b"",
+        )
+        xclient.send_request(req.to_bytes())
+        resp = xclient.recv_response(timeout=2.0)
+
+        assert xserver.is_alive, (
+            "Server crashed - integer truncation in RRChangeOutputProperty (CVE-2023-6478)"
+        )
+        # The server should reject with BadLength (16).  Without the fix
+        # the truncated totalSize (0) passes REQUEST_FIXED_SIZE and the
+        # server tries to allocate 4 GB, failing with BadAlloc (11)
+        # instead.
+        assert isinstance(resp, X11Error), f"Expected an error, got {resp}"
+        assert resp.error_code == 16, (
+            f"Expected BadLength (16), got error code {resp.error_code} - "
+            f"integer truncation not caught by length check"
+        )