From 09eb9aa351a67b24de59cf0cdf33b8c61210d91e Mon Sep 17 00:00:00 2001 From: Jeremy Huddleston Sequoia Date: Sat, 21 Mar 2026 17:09:30 -0700 Subject: [PATCH 1/3] rootless: Fix Glyphs damage bounding box to correctly compute union RootlessGlyphs used max() for box.x1 and box.y1 when accumulating the bounding box across glyphs in a list. Computing the union of bounding boxes requires min() for the lower coordinate corner and max for the higher coordinate corner. Signed-off-by: Jeremy Huddleston Sequoia (cherry picked from commit e8936920ee392086f3af42d86f533bafe589fdb2) --- miext/rootless/rootlessScreen.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miext/rootless/rootlessScreen.c b/miext/rootless/rootlessScreen.c index 79fe3507b..0cf0eda61 100644 --- a/miext/rootless/rootlessScreen.c +++ b/miext/rootless/rootlessScreen.c @@ -332,8 +332,8 @@ RootlessGlyphs(CARD8 op, PicturePtr pSrc, PicturePtr pDst, x2 = x1 + glyph->info.width; y2 = y1 + glyph->info.height; - box.x1 = max(box.x1, x1); - box.y1 = max(box.y1, y1); + box.x1 = min(box.x1, x1); + box.y1 = min(box.y1, y1); box.x2 = max(box.x2, x2); box.y2 = max(box.y2, y2); From d7279b8e4f7317acfa168f8dec61eb205646bd53 Mon Sep 17 00:00:00 2001 From: Jeremy Huddleston Sequoia Date: Sat, 21 Mar 2026 18:02:54 -0700 Subject: [PATCH 2/3] rootless: Add Trapezoids, Triangles, and CompositeRects wrapping The Render extension's Trapezoids and Triangles operations (fbTrapezoids, fbTriangles) render directly via pixman through fbShapes without going through ps->Composite, so RootlessComposite never fires for these operations. This means RootlessStartDrawing is not called and the macOS compositor is not notified of changed pixels. Add RootlessTrapezoids and RootlessTriangles wrappers following the same pattern as RootlessComposite: call RootlessStartDrawing before the operation, then compute the bounding box via miTrapezoidBounds or miTriangleBounds and call RootlessDamageBox after. The default miTriStrip and miTriFan implementations decompose their input into triangles and dispatch through ps->Triangles, so the RootlessTriangles wrapper covers TriStrip and TriFan as well. Also add a RootlessCompositeRects wrapper as defense-in-depth. The current miCompositeRects implementation already has partial rootless coverage: PictOpSrc/PictOpClear go through the GC layer, and other ops route through CompositePicture -> RootlessComposite. Wrapping at the PictureScreen level ensures robustness regardless of the underlying implementation. Fixes [1/2]: https://github.com/XQuartz/XQuartz/issues/31 Signed-off-by: Jeremy Huddleston Sequoia (cherry picked from commit 40b762cf6d4c861ece9a07855d8005e4280e8086) --- miext/rootless/rootlessCommon.h | 3 + miext/rootless/rootlessScreen.c | 138 +++++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/miext/rootless/rootlessCommon.h b/miext/rootless/rootlessCommon.h index 83dec3bb7..e2630b826 100644 --- a/miext/rootless/rootlessCommon.h +++ b/miext/rootless/rootlessCommon.h @@ -108,6 +108,9 @@ typedef struct _RootlessScreenRec { CompositeProcPtr Composite; GlyphsProcPtr Glyphs; + TrapezoidsProcPtr Trapezoids; + TrianglesProcPtr Triangles; + CompositeRectsProcPtr CompositeRects; InstallColormapProcPtr InstallColormap; UninstallColormapProcPtr UninstallColormap; diff --git a/miext/rootless/rootlessScreen.c b/miext/rootless/rootlessScreen.c index 0cf0eda61..3632fbdbf 100644 --- a/miext/rootless/rootlessScreen.c +++ b/miext/rootless/rootlessScreen.c @@ -41,6 +41,7 @@ #include "propertyst.h" #include "mivalidate.h" #include "picturestr.h" +#include "mipict.h" #include "colormapst.h" #include @@ -348,6 +349,135 @@ RootlessGlyphs(CARD8 op, PicturePtr pSrc, PicturePtr pDst, } } +static void +RootlessTrapezoids(CARD8 op, PicturePtr pSrc, PicturePtr pDst, + PictFormatPtr maskFormat, INT16 xSrc, INT16 ySrc, + int ntrap, xTrapezoid *traps) +{ + ScreenPtr pScreen = pDst->pDrawable->pScreen; + PictureScreenPtr ps = GetPictureScreen(pScreen); + WindowPtr srcWin, dstWin; + + srcWin = (pSrc->pDrawable && pSrc->pDrawable->type == DRAWABLE_WINDOW) ? + (WindowPtr) pSrc->pDrawable : NULL; + dstWin = (pDst->pDrawable->type == DRAWABLE_WINDOW) ? + (WindowPtr) pDst->pDrawable : NULL; + + ps->Trapezoids = SCREENREC(pScreen)->Trapezoids; + + if (srcWin && IsFramedWindow(srcWin)) + RootlessStartDrawing(srcWin); + if (dstWin && IsFramedWindow(dstWin)) + RootlessStartDrawing(dstWin); + + ps->Trapezoids(op, pSrc, pDst, maskFormat, xSrc, ySrc, ntrap, traps); + + if (dstWin && IsFramedWindow(dstWin) && ntrap > 0) { + BoxRec box; + + miTrapezoidBounds(ntrap, traps, &box); + + if (box.x1 < box.x2 && box.y1 < box.y2) { + box.x1 += dstWin->drawable.x; + box.y1 += dstWin->drawable.y; + box.x2 += dstWin->drawable.x; + box.y2 += dstWin->drawable.y; + RootlessDamageBox(dstWin, &box); + } + } + + ps->Trapezoids = RootlessTrapezoids; +} + +static void +RootlessTriangles(CARD8 op, PicturePtr pSrc, PicturePtr pDst, + PictFormatPtr maskFormat, INT16 xSrc, INT16 ySrc, + int ntri, xTriangle *tris) +{ + ScreenPtr pScreen = pDst->pDrawable->pScreen; + PictureScreenPtr ps = GetPictureScreen(pScreen); + WindowPtr srcWin, dstWin; + + srcWin = (pSrc->pDrawable && pSrc->pDrawable->type == DRAWABLE_WINDOW) ? + (WindowPtr) pSrc->pDrawable : NULL; + dstWin = (pDst->pDrawable->type == DRAWABLE_WINDOW) ? + (WindowPtr) pDst->pDrawable : NULL; + + ps->Triangles = SCREENREC(pScreen)->Triangles; + + if (srcWin && IsFramedWindow(srcWin)) + RootlessStartDrawing(srcWin); + if (dstWin && IsFramedWindow(dstWin)) + RootlessStartDrawing(dstWin); + + ps->Triangles(op, pSrc, pDst, maskFormat, xSrc, ySrc, ntri, tris); + + if (dstWin && IsFramedWindow(dstWin) && ntri > 0) { + BoxRec box; + + miTriangleBounds(ntri, tris, &box); + + if (box.x1 < box.x2 && box.y1 < box.y2) { + box.x1 += dstWin->drawable.x; + box.y1 += dstWin->drawable.y; + box.x2 += dstWin->drawable.x; + box.y2 += dstWin->drawable.y; + RootlessDamageBox(dstWin, &box); + } + } + + ps->Triangles = RootlessTriangles; +} + +static void +RootlessCompositeRects(CARD8 op, PicturePtr pDst, xRenderColor *color, + int nRect, xRectangle *rects) +{ + ScreenPtr pScreen = pDst->pDrawable->pScreen; + PictureScreenPtr ps = GetPictureScreen(pScreen); + WindowPtr dstWin; + + dstWin = (pDst->pDrawable->type == DRAWABLE_WINDOW) ? + (WindowPtr) pDst->pDrawable : NULL; + + ps->CompositeRects = SCREENREC(pScreen)->CompositeRects; + + if (dstWin && IsFramedWindow(dstWin)) + RootlessStartDrawing(dstWin); + + ps->CompositeRects(op, pDst, color, nRect, rects); + + if (dstWin && IsFramedWindow(dstWin) && nRect > 0) { + int i; + BoxRec box; + + box.x1 = rects[0].x; + box.y1 = rects[0].y; + box.x2 = rects[0].x + rects[0].width; + box.y2 = rects[0].y + rects[0].height; + + for (i = 1; i < nRect; i++) { + short x1 = rects[i].x; + short y1 = rects[i].y; + short x2 = x1 + rects[i].width; + short y2 = y1 + rects[i].height; + + if (x1 < box.x1) box.x1 = x1; + if (y1 < box.y1) box.y1 = y1; + if (x2 > box.x2) box.x2 = x2; + if (y2 > box.y2) box.y2 = y2; + } + + if (box.x1 < box.x2 && box.y1 < box.y2) { + RootlessDamageRect(dstWin, + box.x1, box.y1, + box.x2 - box.x1, box.y2 - box.y1); + } + } + + ps->CompositeRects = RootlessCompositeRects; +} + /* * RootlessValidateTree * ValidateTree is modified in two ways: @@ -685,13 +815,19 @@ RootlessWrap(ScreenPtr pScreen) WRAP(SetShape); { - // Composite and Glyphs don't use normal screen wrapping + // PictureScreen procs don't use normal screen wrapping PictureScreenPtr ps = GetPictureScreen(pScreen); s->Composite = ps->Composite; ps->Composite = RootlessComposite; s->Glyphs = ps->Glyphs; ps->Glyphs = RootlessGlyphs; + s->Trapezoids = ps->Trapezoids; + ps->Trapezoids = RootlessTrapezoids; + s->Triangles = ps->Triangles; + ps->Triangles = RootlessTriangles; + s->CompositeRects = ps->CompositeRects; + ps->CompositeRects = RootlessCompositeRects; } // WRAP(ClearToBackground); fixme put this back? useful for shaped wins? From ef781045eef654189976059b4dad2fbf2f31bc20 Mon Sep 17 00:00:00 2001 From: Jeremy Huddleston Sequoia Date: Sat, 21 Mar 2026 18:03:07 -0700 Subject: [PATCH 3/3] rootless: Protect alpha channel for Render operations The Render extension operates on PictFormats rather than GC, so it bypasses the rootless layer's GC-based alpha protection (ROOTLESS_PROTECT_ALPHA). Depth-24 windows use PICT_x8r8g8b8 where the 'x' tells pixman the high byte is padding it may freely zero. The macOS compositor needs this byte to be 0xFF (opaque). Before each Render operation that targets a rootless window, temporarily upgrade the destination Picture's format from PICT_x8r8g8b8 to PICT_a8r8g8b8. This tells pixman that the alpha channel is significant and must not be optimized away. The format is restored after the operation completes. This parallels how ROOTLESS_PROTECT_ALPHA handles GC operations (by masking alpha out of the planemask) and how ROOTLESS_SAFEALPHA handles PaintWindow (by forcing alpha in solid fills). The save/restore approach is analogous to rootlessGC.c's GC_SAVE / GC_RESTORE pattern. Fixes [2/2]: https://github.com/XQuartz/XQuartz/issues/31 Signed-off-by: Jeremy Huddleston Sequoia (cherry picked from commit d2e83a266c65437902f4a5c85043fa835b5c6013) --- miext/rootless/rootlessScreen.c | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/miext/rootless/rootlessScreen.c b/miext/rootless/rootlessScreen.c index 3632fbdbf..2abe3ad0c 100644 --- a/miext/rootless/rootlessScreen.c +++ b/miext/rootless/rootlessScreen.c @@ -52,6 +52,30 @@ #include "rootlessCommon.h" #include "rootlessWindow.h" +/* + * Render operations use PictFormat to describe pixel layout. Depth-24 + * windows use PICT_x8r8g8b8, where 'x' tells pixman the high byte is + * padding it may freely zero. The compositor needs this byte to be 0xFF + * (opaque). Temporarily upgrading the destination format from 'x' to 'a' + * prevents pixman from optimizing away the alpha channel, paralleling how + * ROOTLESS_PROTECT_ALPHA prevents fb from doing the same for GC ops. + */ + +#if ROOTLESS_PROTECT_ALPHA +#define RL_RENDER_SAVE_FORMAT(pict) \ + CARD32 _saved_format = (pict)->format; \ + if ((pict)->pDrawable->type == DRAWABLE_WINDOW && \ + (pict)->format == PICT_x8r8g8b8) \ + (pict)->format = PICT_a8r8g8b8 + +#define RL_RENDER_RESTORE_FORMAT(pict) \ + (pict)->format = _saved_format + +#else +#define RL_RENDER_SAVE_FORMAT(pict) +#define RL_RENDER_RESTORE_FORMAT(pict) +#endif + extern int RootlessMiValidateTree(WindowPtr pRoot, WindowPtr pChild, VTKind kind); extern Bool RootlessCreateGC(GCPtr pGC); @@ -258,8 +282,10 @@ RootlessComposite(CARD8 op, PicturePtr pSrc, PicturePtr pMask, PicturePtr pDst, if (dstWin && IsFramedWindow(dstWin)) RootlessStartDrawing(dstWin); + RL_RENDER_SAVE_FORMAT(pDst); ps->Composite(op, pSrc, pMask, pDst, xSrc, ySrc, xMask, yMask, xDst, yDst, width, height); + RL_RENDER_RESTORE_FORMAT(pDst); if (dstWin && IsFramedWindow(dstWin)) { RootlessDamageRect(dstWin, xDst, yDst, width, height); @@ -293,7 +319,9 @@ RootlessGlyphs(CARD8 op, PicturePtr pSrc, PicturePtr pDst, //SCREEN_UNWRAP(ps, Glyphs); ps->Glyphs = SCREENREC(pScreen)->Glyphs; + RL_RENDER_SAVE_FORMAT(pDst); ps->Glyphs(op, pSrc, pDst, maskFormat, xSrc, ySrc, nlist, list, glyphs); + RL_RENDER_RESTORE_FORMAT(pDst); ps->Glyphs = RootlessGlyphs; //SCREEN_WRAP(ps, Glyphs); @@ -370,7 +398,9 @@ RootlessTrapezoids(CARD8 op, PicturePtr pSrc, PicturePtr pDst, if (dstWin && IsFramedWindow(dstWin)) RootlessStartDrawing(dstWin); + RL_RENDER_SAVE_FORMAT(pDst); ps->Trapezoids(op, pSrc, pDst, maskFormat, xSrc, ySrc, ntrap, traps); + RL_RENDER_RESTORE_FORMAT(pDst); if (dstWin && IsFramedWindow(dstWin) && ntrap > 0) { BoxRec box; @@ -410,7 +440,9 @@ RootlessTriangles(CARD8 op, PicturePtr pSrc, PicturePtr pDst, if (dstWin && IsFramedWindow(dstWin)) RootlessStartDrawing(dstWin); + RL_RENDER_SAVE_FORMAT(pDst); ps->Triangles(op, pSrc, pDst, maskFormat, xSrc, ySrc, ntri, tris); + RL_RENDER_RESTORE_FORMAT(pDst); if (dstWin && IsFramedWindow(dstWin) && ntri > 0) { BoxRec box; @@ -445,7 +477,9 @@ RootlessCompositeRects(CARD8 op, PicturePtr pDst, xRenderColor *color, if (dstWin && IsFramedWindow(dstWin)) RootlessStartDrawing(dstWin); + RL_RENDER_SAVE_FORMAT(pDst); ps->CompositeRects(op, pDst, color, nRect, rects); + RL_RENDER_RESTORE_FORMAT(pDst); if (dstWin && IsFramedWindow(dstWin) && nRect > 0) { int i;