From 8237b60ec38e5caae28a0f3e7c8cd5c2be4a3c24 Mon Sep 17 00:00:00 2001
From: Jeremy Huddleston Sequoia <jeremyhu@apple.com>
Date: Sat, 21 Mar 2026 18:03:07 -0700
Subject: [PATCH] rootless: Protect alpha channel for Render operations

The Render extension operates on PictFormats rather than GC, so it
bypasses the rootless layer's GC-based alpha protection
(ROOTLESS_PROTECT_ALPHA).  Depth-24 windows use PICT_x8r8g8b8 where
the 'x' tells pixman the high byte is padding it may freely zero.
The macOS compositor needs this byte to be 0xFF (opaque).

Before each Render operation that targets a rootless window,
temporarily upgrade the destination Picture's format from
PICT_x8r8g8b8 to PICT_a8r8g8b8.  This tells pixman that the alpha
channel is significant and must not be optimized away.  The format
is restored after the operation completes.

This parallels how ROOTLESS_PROTECT_ALPHA handles GC operations (by
masking alpha out of the planemask) and how ROOTLESS_SAFEALPHA
handles PaintWindow (by forcing alpha in solid fills).  The
save/restore approach is analogous to rootlessGC.c's GC_SAVE /
GC_RESTORE pattern.

Fixes [2/2]: https://github.com/XQuartz/XQuartz/issues/31

Signed-off-by: Jeremy Huddleston Sequoia <jeremyhu@apple.com>
---
 miext/rootless/rootlessScreen.c | 34 +++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/miext/rootless/rootlessScreen.c b/miext/rootless/rootlessScreen.c
index 6f6ab89b4..1d8a3a21c 100644
--- a/miext/rootless/rootlessScreen.c
+++ b/miext/rootless/rootlessScreen.c
@@ -52,6 +52,30 @@
 #include "rootlessCommon.h"
 #include "rootlessWindow.h"
 
+/*
+ * Render operations use PictFormat to describe pixel layout.  Depth-24
+ * windows use PICT_x8r8g8b8, where 'x' tells pixman the high byte is
+ * padding it may freely zero.  The compositor needs this byte to be 0xFF
+ * (opaque).  Temporarily upgrading the destination format from 'x' to 'a'
+ * prevents pixman from optimizing away the alpha channel, paralleling how
+ * ROOTLESS_PROTECT_ALPHA prevents fb from doing the same for GC ops.
+ */
+
+#if ROOTLESS_PROTECT_ALPHA
+#define RL_RENDER_SAVE_FORMAT(pict)                              \
+    CARD32 _saved_format = (pict)->format;                       \
+    if ((pict)->pDrawable->type == DRAWABLE_WINDOW &&            \
+        (pict)->format == PICT_x8r8g8b8)                         \
+        (pict)->format = PICT_a8r8g8b8
+
+#define RL_RENDER_RESTORE_FORMAT(pict) \
+    (pict)->format = _saved_format
+
+#else
+#define RL_RENDER_SAVE_FORMAT(pict)
+#define RL_RENDER_RESTORE_FORMAT(pict)
+#endif
+
 extern int RootlessMiValidateTree(WindowPtr pRoot, WindowPtr pChild,
                                   VTKind kind);
 extern Bool RootlessCreateGC(GCPtr pGC);
@@ -258,8 +282,10 @@ RootlessComposite(CARD8 op, PicturePtr pSrc, PicturePtr pMask, PicturePtr pDst,
     if (dstWin && IsFramedWindow(dstWin))
         RootlessStartDrawing(dstWin);
 
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->Composite(op, pSrc, pMask, pDst,
                   xSrc, ySrc, xMask, yMask, xDst, yDst, width, height);
+    RL_RENDER_RESTORE_FORMAT(pDst);
 
     if (dstWin && IsFramedWindow(dstWin)) {
         RootlessDamageRect(dstWin, xDst, yDst, width, height);
@@ -293,7 +319,9 @@ RootlessGlyphs(CARD8 op, PicturePtr pSrc, PicturePtr pDst,
 
     //SCREEN_UNWRAP(ps, Glyphs);
     ps->Glyphs = SCREENREC(pScreen)->Glyphs;
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->Glyphs(op, pSrc, pDst, maskFormat, xSrc, ySrc, nlist, list, glyphs);
+    RL_RENDER_RESTORE_FORMAT(pDst);
     ps->Glyphs = RootlessGlyphs;
     //SCREEN_WRAP(ps, Glyphs);
 
@@ -375,7 +403,9 @@ RootlessTrapezoids(CARD8 op, PicturePtr pSrc, PicturePtr pDst,
     if (dstWin && IsFramedWindow(dstWin))
         RootlessStartDrawing(dstWin);
 
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->Trapezoids(op, pSrc, pDst, maskFormat, xSrc, ySrc, ntrap, traps);
+    RL_RENDER_RESTORE_FORMAT(pDst);
 
     if (dstWin && IsFramedWindow(dstWin) && ntrap > 0) {
         BoxRec box;
@@ -415,7 +445,9 @@ RootlessTriangles(CARD8 op, PicturePtr pSrc, PicturePtr pDst,
     if (dstWin && IsFramedWindow(dstWin))
         RootlessStartDrawing(dstWin);
 
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->Triangles(op, pSrc, pDst, maskFormat, xSrc, ySrc, ntri, tris);
+    RL_RENDER_RESTORE_FORMAT(pDst);
 
     if (dstWin && IsFramedWindow(dstWin) && ntri > 0) {
         BoxRec box;
@@ -450,7 +482,9 @@ RootlessCompositeRects(CARD8 op, PicturePtr pDst, xRenderColor *color,
     if (dstWin && IsFramedWindow(dstWin))
         RootlessStartDrawing(dstWin);
 
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->CompositeRects(op, pDst, color, nRect, rects);
+    RL_RENDER_RESTORE_FORMAT(pDst);
 
     if (dstWin && IsFramedWindow(dstWin) && nRect > 0) {
         int i;