diff --git a/boilerplate/cairo-boilerplate.c b/boilerplate/cairo-boilerplate.c
index b8cad0309..e5257663d 100755
--- a/boilerplate/cairo-boilerplate.c
+++ b/boilerplate/cairo-boilerplate.c
@@ -93,11 +93,12 @@ _cairo_test_content_name (cairo_content_t content)
 }
 
 static cairo_surface_t *
-create_image_surface (const char	 *name,
-		      cairo_content_t	  content,
-		      int		  width,
-		      int		  height,
-		      void		**closure)
+create_image_surface (const char		 *name,
+		      cairo_content_t		  content,
+		      int			  width,
+		      int			  height,
+		      cairo_boilerplate_mode_t	  mode,
+		      void			**closure)
 {
     cairo_format_t format;
     *closure = NULL;
@@ -121,22 +122,24 @@ create_image_surface (const char	 *name,
 #include "test-paginated-surface.h"
 
 static cairo_surface_t *
-create_test_fallback_surface (const char	 *name,
-			      cairo_content_t	  content,
-			      int		  width,
-			      int		  height,
-			      void		**closure)
+create_test_fallback_surface (const char		 *name,
+			      cairo_content_t		  content,
+			      int			  width,
+			      int			  height,
+			      cairo_boilerplate_mode_t	  mode,
+			      void			**closure)
 {
     *closure = NULL;
     return _test_fallback_surface_create (content, width, height);
 }
 
 static cairo_surface_t *
-create_test_meta_surface (const char		 *name,
-			  cairo_content_t	  content,
-			  int			  width,
-			  int			  height,
-			  void			**closure)
+create_test_meta_surface (const char			 *name,
+			  cairo_content_t		  content,
+			  int				  width,
+			  int				  height,
+			  cairo_boilerplate_mode_t	  mode,
+			  void				**closure)
 {
     *closure = NULL;
     return _test_meta_surface_create (content, width, height);
@@ -153,11 +156,12 @@ typedef struct {
 } test_paginated_closure_t;
 
 static cairo_surface_t *
-create_test_paginated_surface (const char	 *name,
-			       cairo_content_t	  content,
-			       int		  width,
-			       int		  height,
-			       void		**closure)
+create_test_paginated_surface (const char		 *name,
+			       cairo_content_t		  content,
+			       int			  width,
+			       int			  height,
+			       cairo_boilerplate_mode_t	  mode,
+			       void			**closure)
 {
     test_paginated_closure_t *tpc;
     cairo_surface_t *surface;
@@ -374,11 +378,12 @@ create_glitz_glx_surface (glitz_format_name_t	      formatname,
 }
 
 static cairo_surface_t *
-create_cairo_glitz_glx_surface (const char	 *name,
-				cairo_content_t   content,
-				int		  width,
-				int		  height,
-				void		**closure)
+create_cairo_glitz_glx_surface (const char			 *name,
+				cairo_content_t 		  content,
+				int				  width,
+				int				  height,
+				cairo_boilerplate_mode_t	  mode,
+				void				**closure)
 {
     glitz_glx_target_closure_t *gxtc;
     glitz_surface_t  * glitz_surface;
@@ -518,11 +523,12 @@ create_glitz_agl_surface (glitz_format_name_t formatname,
 }
 
 static cairo_surface_t *
-create_cairo_glitz_agl_surface (const char	 *name,
-				cairo_content_t   content,
-				int		  width,
-				int		  height,
-				void		**closure)
+create_cairo_glitz_agl_surface (const char			 *name,
+				cairo_content_t 		  content,
+				int				  width,
+				int				  height,
+				cairo_boilerplate_mode_t	  mode,
+				void				**closure)
 {
     glitz_surface_t *glitz_surface;
     cairo_surface_t *surface;
@@ -636,11 +642,12 @@ create_glitz_wgl_surface (glitz_format_name_t formatname,
 }
 
 static cairo_surface_t *
-create_cairo_glitz_wgl_surface (const char	 *name,
-				cairo_content_t	  content,
-				int		  width,
-				int		  height,
-				void		**closure)
+create_cairo_glitz_wgl_surface (const char			 *name,
+				cairo_content_t			  content,
+				int				  width,
+				int				  height,
+				cairo_boilerplate_mode_t	  mode,
+				void				**closure)
 {
     glitz_surface_t *glitz_surface;
     cairo_surface_t *surface;
@@ -709,11 +716,12 @@ cleanup_quartz (void *closure)
 #if CAIRO_HAS_WIN32_SURFACE
 #include "cairo-win32.h"
 static cairo_surface_t *
-create_win32_surface (const char	 *name,
-		      cairo_content_t	  content,
-		      int		  width,
-		      int		  height,
-		      void		**closure)
+create_win32_surface (const char		 *name,
+		      cairo_content_t		  content,
+		      int			  width,
+		      int			  height,
+		      cairo_boilerplate_mode_t	  mode,
+		      void			**closure)
 {
     cairo_format_t format;
 
@@ -832,11 +840,12 @@ _format_from_cairo(XCBConnection *c, cairo_format_t fmt)
 }
 
 static cairo_surface_t *
-create_xcb_surface (const char		 *name,
-		    cairo_content_t	  content,
-		    int			  width,
-		    int			  height,
-		    void		**closure)
+create_xcb_surface (const char			 *name,
+		    cairo_content_t		  content,
+		    int				  width,
+		    int				  height,
+		    cairo_boilerplate_mode_t	  mode,
+		    void			**closure)
 {
     XCBSCREEN *root;
     xcb_target_closure_t *xtc;
@@ -906,34 +915,33 @@ cleanup_xcb (void *closure)
 typedef struct _xlib_target_closure
 {
     Display *dpy;
-    Pixmap pixmap;
+    Drawable drawable;
+    cairo_bool_t drawable_is_pixmap;
 } xlib_target_closure_t;
 
+/* For the xlib backend we distinguish between TEST and PERF mode in a
+ * couple of ways.
+ *
+ * For TEST, we always test against pixmaps of depth 32 (for
+ * COLOR_ALPHA) or 24 (for COLOR) and we use XSynchronize to make it
+ * easier to debug problems.
+ *
+ * For PERF, we test against 32-bit pixmaps for COLOR_ALPHA, but for
+ * COLOR we test against _windows_ at the depth of the default visual.
+ * For obvious reasons, we don't use XSynchronize.
+ */
 static cairo_surface_t *
-create_xlib_surface (const char		 *name,
-		     cairo_content_t	  content,
-		     int		  width,
-		     int		  height,
-		     void		**closure)
+create_xlib_test_surface (Display		*dpy,
+			  cairo_content_t	 content,
+			  int			 width,
+			  int			 height,
+			  xlib_target_closure_t	*xtc)
 {
-    xlib_target_closure_t *xtc;
-    cairo_surface_t *surface;
-    Display *dpy;
     XRenderPictFormat *xrender_format;
 
-    *closure = xtc = xmalloc (sizeof (xlib_target_closure_t));
-
-    if (width == 0)
-	width = 1;
-    if (height == 0)
-	height = 1;
-
-    xtc->dpy = dpy = XOpenDisplay (NULL);
-    if (xtc->dpy == NULL) {
-	CAIRO_BOILERPLATE_LOG ("Failed to open display: %s\n", XDisplayName(0));
-	return NULL;
-    }
-
+    /* This kills performance, but it makes debugging much
+     * easier. That's why we have it here when in TEST mode, but not
+     * over in PERF mode. */
     XSynchronize (xtc->dpy, 1);
 
     /* XXX: Currently we don't do any xlib testing when the X server
@@ -961,14 +969,88 @@ create_xlib_surface (const char		 *name,
 	return NULL;
     }
 
-    xtc->pixmap = XCreatePixmap (dpy, DefaultRootWindow (dpy),
-				 width, height, xrender_format->depth);
+    xtc->drawable = XCreatePixmap (dpy, DefaultRootWindow (dpy),
+				   width, height, xrender_format->depth);
+    xtc->drawable_is_pixmap = TRUE;
 
-    surface = cairo_xlib_surface_create_with_xrender_format (dpy, xtc->pixmap,
-							     DefaultScreenOfDisplay (dpy),
-							     xrender_format,
-							     width, height);
-    return surface;
+    return cairo_xlib_surface_create_with_xrender_format (dpy, xtc->drawable,
+							  DefaultScreenOfDisplay (dpy),
+							  xrender_format,
+							  width, height);
+}
+
+static cairo_surface_t *
+create_xlib_perf_surface (Display		*dpy,
+			  cairo_content_t	 content,
+			  int			 width,
+			  int			 height,
+			  xlib_target_closure_t	*xtc)
+{
+    XSetWindowAttributes attr;
+    XRenderPictFormat *xrender_format;
+    Visual *visual;
+
+    switch (content) {
+    case CAIRO_CONTENT_COLOR_ALPHA:
+	xrender_format = XRenderFindStandardFormat (dpy, PictStandardARGB32);
+	xtc->drawable = XCreatePixmap (dpy, DefaultRootWindow (dpy),
+				       width, height, xrender_format->depth);
+	xtc->drawable_is_pixmap = TRUE;
+	break;
+    case CAIRO_CONTENT_COLOR:
+	visual = DefaultVisual (dpy, DefaultScreen (dpy));
+	xrender_format = XRenderFindVisualFormat (dpy, visual);
+	attr.override_redirect = True;
+	xtc->drawable = XCreateWindow (dpy, DefaultRootWindow (dpy), 0, 0,
+				       width, height, 0, xrender_format->depth,
+				       InputOutput, visual, CWOverrideRedirect, &attr);
+	XMapWindow (dpy, xtc->drawable);
+	xtc->drawable_is_pixmap = FALSE;
+	break;
+    case CAIRO_CONTENT_ALPHA:
+    default:
+	CAIRO_BOILERPLATE_LOG ("Invalid content for xlib test: %d\n", content);
+	return NULL;
+    }
+    if (xrender_format == NULL) {
+	CAIRO_BOILERPLATE_LOG ("X server does not have the Render extension.\n");
+	return NULL;
+    }
+
+    return cairo_xlib_surface_create_with_xrender_format (dpy, xtc->drawable,
+							  DefaultScreenOfDisplay (dpy),
+							  xrender_format,
+							  width, height);
+}
+
+static cairo_surface_t *
+create_xlib_surface (const char			 *name,
+		     cairo_content_t		  content,
+		     int			  width,
+		     int			  height,
+		     cairo_boilerplate_mode_t	  mode,
+		     void			**closure)
+{
+    xlib_target_closure_t *xtc;
+    Display *dpy;
+
+    *closure = xtc = xmalloc (sizeof (xlib_target_closure_t));
+
+    if (width == 0)
+	width = 1;
+    if (height == 0)
+	height = 1;
+
+    xtc->dpy = dpy = XOpenDisplay (NULL);
+    if (xtc->dpy == NULL) {
+	CAIRO_BOILERPLATE_LOG ("Failed to open display: %s\n", XDisplayName(0));
+	return NULL;
+    }
+
+    if (mode == CAIRO_BOILERPLATE_MODE_TEST)
+	return create_xlib_test_surface (dpy, content, width, height, xtc);
+    else /* mode == CAIRO_BOILERPLATE_MODE_PERF */
+	return create_xlib_perf_surface (dpy, content, width, height, xtc);
 }
 
 static void
@@ -976,7 +1058,10 @@ cleanup_xlib (void *closure)
 {
     xlib_target_closure_t *xtc = closure;
 
-    XFreePixmap (xtc->dpy, xtc->pixmap);
+    if (xtc->drawable_is_pixmap)
+	XFreePixmap (xtc->dpy, xtc->drawable);
+    else
+	XDestroyWindow (xtc->dpy, xtc->drawable);
     XCloseDisplay (xtc->dpy);
     free (xtc);
 }
@@ -1005,11 +1090,12 @@ typedef struct _ps_target_closure
 } ps_target_closure_t;
 
 static cairo_surface_t *
-create_ps_surface (const char		 *name,
-		   cairo_content_t	  content,
-		   int			  width,
-		   int			  height,
-		   void			**closure)
+create_ps_surface (const char			 *name,
+		   cairo_content_t		  content,
+		   int				  width,
+		   int				  height,
+		   cairo_boilerplate_mode_t	  mode,
+		   void				**closure)
 {
     ps_target_closure_t	*ptc;
     cairo_surface_t *surface;
@@ -1113,11 +1199,12 @@ typedef struct _pdf_target_closure
 } pdf_target_closure_t;
 
 static cairo_surface_t *
-create_pdf_surface (const char		 *name,
-		    cairo_content_t	  content,
-		    int			  width,
-		    int			  height,
-		    void		**closure)
+create_pdf_surface (const char			 *name,
+		    cairo_content_t		  content,
+		    int				  width,
+		    int				  height,
+		    cairo_boilerplate_mode_t	  mode,
+		    void			**closure)
 {
     pdf_target_closure_t *ptc;
     cairo_surface_t *surface;
@@ -1222,11 +1309,12 @@ typedef struct _svg_target_closure
 } svg_target_closure_t;
 
 static cairo_surface_t *
-create_svg_surface (const char		 *name,
-		    cairo_content_t	  content,
-		    int			  width,
-		    int			  height,
-		    void		**closure)
+create_svg_surface (const char			 *name,
+		    cairo_content_t		  content,
+		    int				  width,
+		    int				  height,
+		    cairo_boilerplate_mode_t	  mode,
+		    void			**closure)
 {
     int i;
     svg_target_closure_t *ptc;
diff --git a/boilerplate/cairo-boilerplate.h b/boilerplate/cairo-boilerplate.h
index b40825f35..a75bce9dd 100644
--- a/boilerplate/cairo-boilerplate.h
+++ b/boilerplate/cairo-boilerplate.h
@@ -88,12 +88,18 @@ _cairo_test_content_name (cairo_content_t content);
 #define TRUE 1
 #endif
 
+typedef enum {
+    CAIRO_BOILERPLATE_MODE_TEST,
+    CAIRO_BOILERPLATE_MODE_PERF
+} cairo_boilerplate_mode_t;
+
 typedef cairo_surface_t *
-(*cairo_boilerplate_create_surface_t) (const char	 *name,
-				       cairo_content_t	  content,
-				       int		  width,
-				       int		  height,
-				       void		**closure);
+(*cairo_boilerplate_create_surface_t) (const char		 *name,
+				       cairo_content_t		  content,
+				       int			  width,
+				       int			  height,
+				       cairo_boilerplate_mode_t	  mode,
+				       void			**closure);
 
 typedef cairo_status_t
 (*cairo_boilerplate_write_to_png_t) (cairo_surface_t *surface, const char *filename);
diff --git a/perf/cairo-perf.c b/perf/cairo-perf.c
index 4a2147ae0..615acc940 100644
--- a/perf/cairo-perf.c
+++ b/perf/cairo-perf.c
@@ -154,14 +154,13 @@ main (int argc, char *argv[])
 	    continue;
 	if (cairo_test_target && ! strstr (cairo_test_target, target->name))
 	    continue;
-	if (target->content == CAIRO_CONTENT_COLOR)
-	    continue;
 	for (j = 0; perfs[j].name; j++) {
 	    perf = &perfs[j];
 	    for (size = perf->min_size; size <= perf->max_size; size *= 2) {
 		surface = (target->create_surface) (perf->name,
 						    target->content,
 						    size, size,
+						    CAIRO_BOILERPLATE_MODE_PERF,
 						    &target->closure);
 		cr = cairo_create (surface);
 		for (k =0; k < cairo_perf_iterations; k++) {
diff --git a/test/cairo-test.c b/test/cairo-test.c
index 36a40cf95..d95d15004 100644
--- a/test/cairo-test.c
+++ b/test/cairo-test.c
@@ -220,6 +220,7 @@ cairo_test_for_target (cairo_test_t			 *test,
 					target->content,
 					test->width,
 					test->height,
+					CAIRO_BOILERPLATE_MODE_TEST,
 					&target->closure);
 
     if (test->width && test->height) {