From 02236e3fda2cbae61e87d679ce59fed8d5b0d08d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michel=20D=C3=A4nzer?= <mdaenzer@redhat.com>
Date: Fri, 5 Jan 2024 18:31:11 +0100
Subject: [PATCH] glamor: Don't override source alpha to 1.0 if it's used for
 blending

It caused an incorrect result of the blend operation.

Use glColorMask to prevent non-1.0 alpha channel values in a depth 32
pixmap backing an effective depth 24 window. For blending operations,
the expectation is that the destination drawable contains valid pixel
values, so the alpha channel should already be 1.0.

Fixes: d1f142891ef3 ("glamor: Ignore destination alpha as necessary for composite operation")
Issue: https://gitlab.gnome.org/GNOME/mutter/-/issues/3104
(cherry picked from commit d1bbf82d72566e16e90800c77cf70fe5fc4ef2e8)
---
 glamor/glamor_render.c | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/glamor/glamor_render.c b/glamor/glamor_render.c
index 0d233f27b..60e1c64ac 100644
--- a/glamor/glamor_render.c
+++ b/glamor/glamor_render.c
@@ -834,6 +834,20 @@ glamor_render_format_is_supported(PicturePtr picture)
     }
 }
 
+static Bool
+render_op_uses_src_alpha(CARD8 op)
+{
+    struct blendinfo *info = &composite_op_info[op];
+
+    switch (info->dest_blend) {
+    case GL_ONE_MINUS_SRC_ALPHA:
+    case GL_SRC_ALPHA:
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
 static Bool
 glamor_composite_choose_shader(CARD8 op,
                                PicturePtr source,
@@ -947,7 +961,8 @@ glamor_composite_choose_shader(CARD8 op,
         key.dest_swizzle = SHADER_DEST_SWIZZLE_ALPHA_TO_RED;
     } else {
         if (dest_pixmap->drawable.depth == 32 &&
-            glamor_drawable_effective_depth(dest->pDrawable) == 24)
+            glamor_drawable_effective_depth(dest->pDrawable) == 24 &&
+            !render_op_uses_src_alpha(op))
             key.dest_swizzle = SHADER_DEST_SWIZZLE_IGNORE_ALPHA;
         else
             key.dest_swizzle = SHADER_DEST_SWIZZLE_DEFAULT;
@@ -1181,6 +1196,7 @@ glamor_composite_with_shader(CARD8 op,
     Bool ret = FALSE;
     glamor_composite_shader *shader = NULL, *shader_ca = NULL;
     struct blendinfo op_info, op_info_ca;
+    Bool restore_colormask = FALSE;
 
     if (!glamor_composite_choose_shader(op, source, mask, dest,
                                         source_pixmap, mask_pixmap, dest_pixmap,
@@ -1205,6 +1221,14 @@ glamor_composite_with_shader(CARD8 op,
 
     glamor_make_current(glamor_priv);
 
+    if (ca_state != CA_TWO_PASS &&
+        key.dest_swizzle == SHADER_DEST_SWIZZLE_DEFAULT &&
+        dest_pixmap->drawable.depth == 32 &&
+        glamor_drawable_effective_depth(dest->pDrawable) == 24) {
+        glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_FALSE);
+        restore_colormask = TRUE;
+    }
+
     glamor_set_destination_pixmap_priv_nc(glamor_priv, dest_pixmap, dest_pixmap_priv);
     glamor_composite_set_shader_blend(glamor_priv, dest_pixmap_priv, &key, shader, &op_info);
     glamor_set_alu(screen, GXcopy);
@@ -1347,6 +1371,8 @@ glamor_composite_with_shader(CARD8 op,
 
     glDisable(GL_SCISSOR_TEST);
 disable_va:
+    if (restore_colormask)
+        glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
     glDisableVertexAttribArray(GLAMOR_VERTEX_POS);
     glDisableVertexAttribArray(GLAMOR_VERTEX_SOURCE);
     glDisableVertexAttribArray(GLAMOR_VERTEX_MASK);