[test] Track XFAIL using expected results stored as xfail.png

Instead of tagging the sources, which is insensitive to changes, track the
known failure modes by recording the current fail as an xfail.png
reference. (We also introduce a new.png to track a fresh error, so that
they are not lost in the noise of the old XFAILs and hopefully do not
cause everyone to fret).

As we have removed the XFAIL tagging we find, surprise surprise, that some
tests are now working -- so review all the reference images (as also some
.ref.png now should be .xfail.png).

Note: I've only checked image,pdf,ps,svg. The test surfaces report some
failures that probably need to addressed in source. I've not correct the
changes for win32 and quartz. Nor fixed up the experimental backends.

test/alpha-similar.c

@@ -63,7 +63,7 @@ draw (cairo_t *cr, int width, int height)
 CAIRO_TEST (alpha_similar,
 	    "Tests creation of similar alpha surfaces"
 	    "\nApplication of a pure-alpha similar source is inconsistent across backends.",
-	    "XFAIL=ps,pdf,svg alpha, similar", /* keywords */
+	    "alpha, similar", /* keywords */
 	    NULL, /* requirements */
 	    10, 10,
 	    NULL, draw)

test/big-line.c

@@ -56,7 +56,7 @@ draw (cairo_t *cr, int width, int height)
 CAIRO_TEST (big_line,
 	    "Test drawing of simple lines with positive and negative coordinates > 2^16\n"
 	    "This currently fails because of 16-bit limitations in pixman.",
-	    "XFAIL line", /* keywords */
+	    "line", /* keywords */
 	    NULL, /* requirements */
 	    100, 100,
 	    NULL, draw)

test/big-trap.c

@@ -61,6 +61,11 @@
 static cairo_test_status_t
 draw (cairo_t *cr, int width, int height)
 {
+    cairo_set_source_rgb (cr, 1,1,1);
+    cairo_paint (cr);
+
+    cairo_set_source_rgb (cr, 0,0,0);
+
     /* Note that without the clip, this doesn't crash... */
     cairo_new_path (cr);
     cairo_rectangle (cr, 0, 0, width, height);
@@ -80,7 +85,7 @@ draw (cairo_t *cr, int width, int height)
 CAIRO_TEST (big_trap,
 	    "Test oversize trapezoid with a clip region"
 	    "\nTest needs to be adjusted to trigger the original bug",
-	    "XFAIL trap", /* keywords */
+	    "trap", /* keywords */
 	    NULL, /* requirements */
 	    100, 100,
 	    NULL, draw)

test/cairo-test-runner.c

@@ -66,7 +66,6 @@ typedef struct _cairo_test_runner {
 
     cairo_bool_t passed;
     int num_passed;
-    int num_xpassed;
     int num_skipped;
     int num_failed;
     int num_xfailed;
@@ -74,11 +73,9 @@ typedef struct _cairo_test_runner {
 
     cairo_test_list_t **crashes_per_target;
     cairo_test_list_t **fails_per_target;
-    cairo_test_list_t **xpasses_per_target;
 
     int *num_failed_per_target;
     int *num_crashed_per_target;
-    int *num_xpassed_per_target;
 
     cairo_bool_t foreground;
     cairo_bool_t list_only;
@@ -342,14 +339,10 @@ _runner_init (cairo_test_runner_t *runner)
 
     runner->passed = TRUE;
 
-    runner->xpasses_per_target = xcalloc (sizeof (cairo_test_list_t *),
-					  runner->base.num_targets);
     runner->fails_per_target = xcalloc (sizeof (cairo_test_list_t *),
 					runner->base.num_targets);
     runner->crashes_per_target = xcalloc (sizeof (cairo_test_list_t *),
 					  runner->base.num_targets);
-    runner->num_xpassed_per_target = xcalloc (sizeof (int),
-					      runner->base.num_targets);
     runner->num_failed_per_target = xcalloc (sizeof (int),
 					     runner->base.num_targets);
     runner->num_crashed_per_target = xcalloc (sizeof (int),
@@ -373,9 +366,8 @@ static void
 _runner_print_summary (cairo_test_runner_t *runner)
 {
     _log (&runner->base,
-	  "%d Passed [%d unexpectedly], %d Failed [%d crashed, %d expected], %d Skipped\n",
+	  "%d Passed, %d Failed [%d crashed, %d expected], %d Skipped\n",
-	  runner->num_passed + runner->num_xpassed,
+	  runner->num_passed,
-	  runner->num_xpassed,
 
 	  runner->num_failed + runner->num_crashed + runner->num_xfailed,
 	  runner->num_crashed,
@@ -427,23 +419,6 @@ _runner_print_details (cairo_test_runner_t *runner)
 	    }
 	    _log (&runner->base, "\n");
 	}
-
-	if (runner->num_xpassed_per_target[n]) {
-	    _log (&runner->base, "%s (%s): %d unexpectedly passed -",
-		  target->name,
-		  cairo_boilerplate_content_name (target->content),
-		  runner->num_xpassed_per_target[n]);
-
-	    for (list = runner->xpasses_per_target[n];
-		 list != NULL;
-		 list = list->next)
-	    {
-		char *name = cairo_test_get_name (list->test);
-		_log (&runner->base, " %s", name);
-		free (name);
-	    }
-	    _log (&runner->base, "\n");
-	}
     }
 }
 
@@ -469,21 +444,18 @@ _runner_fini (cairo_test_runner_t *runner)
     for (n = 0; n < runner->base.num_targets; n++) {
 	_list_free (runner->crashes_per_target[n]);
 	_list_free (runner->fails_per_target[n]);
-	_list_free (runner->xpasses_per_target[n]);
     }
     free (runner->crashes_per_target);
     free (runner->fails_per_target);
-    free (runner->xpasses_per_target);
 
     free (runner->num_crashed_per_target);
     free (runner->num_failed_per_target);
-    free (runner->num_xpassed_per_target);
 
     cairo_test_fini (&runner->base);
 
     return runner->num_failed + runner->num_crashed ?
 	CAIRO_TEST_FAILURE :
-	runner->num_passed + runner->num_xpassed + runner->num_xfailed ?
+	runner->num_passed + runner->num_xfailed ?
 	CAIRO_TEST_SUCCESS : CAIRO_TEST_UNTESTED;
 }
 
@@ -632,9 +604,8 @@ main (int argc, char **argv)
 
     for (list = tests; list != NULL; list = list->next) {
 	cairo_test_context_t ctx;
-	cairo_test_status_t status, expectation;
+	cairo_test_status_t status;
-	cairo_bool_t failed = FALSE, crashed = FALSE, skipped = TRUE;
+	cairo_bool_t failed = FALSE, xfailed = FALSE, crashed = FALSE, skipped = TRUE;
-	const char *XFAIL = NULL;
 	char *name = cairo_test_get_name (list->test);
 	int i;
 
@@ -657,7 +628,7 @@ main (int argc, char **argv)
 		    found = TRUE;
 		}
 
-		/* keyword match */
+		/* XXX keyword match */
 		if (keywords != NULL && strstr (keywords, match) != NULL) {
 		    found = ! invert;
 		    break;
@@ -715,20 +686,6 @@ main (int argc, char **argv)
 	    goto TEST_NEXT;
 	}
 
-	/* check for a failing test */
-	expectation = CAIRO_TEST_SUCCESS;
-	if (list->test->keywords != NULL &&
-	    (XFAIL = strstr (list->test->keywords, "XFAIL")) != NULL)
-	{
-	    if (XFAIL[5] == '=') {
-		/* backend specific expected failure */
-		XFAIL += 5;
-	    } else {
-		XFAIL = NULL;
-		expectation = CAIRO_TEST_FAILURE;
-	    }
-	}
-
 	_cairo_test_context_init_for_test (&ctx, &runner.base, list->test);
 	memset (target_status, 0,
 		sizeof (cairo_test_status_t) * ctx.num_targets);
@@ -739,10 +696,14 @@ main (int argc, char **argv)
 	    case CAIRO_TEST_SUCCESS:
 		skipped = FALSE;
 		break;
-	    case CAIRO_TEST_NO_MEMORY:
+	    case CAIRO_TEST_XFAILURE:
+		xfailed = TRUE;
+		goto TEST_DONE;
+	    case CAIRO_TEST_NEW:
 	    case CAIRO_TEST_FAILURE:
 		failed = TRUE;
 		goto TEST_DONE;
+	    case CAIRO_TEST_NO_MEMORY:
 	    case CAIRO_TEST_CRASHED:
 		failed = TRUE;
 		goto TEST_DONE;
@@ -757,35 +718,13 @@ main (int argc, char **argv)
 	for (n = 0; n < ctx.num_targets; n++) {
 	    const cairo_boilerplate_target_t *target;
 	    cairo_bool_t target_failed = FALSE,
+			 target_xfailed = FALSE,
 			 target_crashed = FALSE,
 			 target_skipped = TRUE;
 	    int has_similar;
 
 	    target = ctx.targets_to_test[n];
 
-	    if (XFAIL != NULL) {
-		const char *target_XFAIL, *base_XFAIL = NULL;
-
-		if (((target_XFAIL = strstr (XFAIL, target->name)) != NULL ||
-		     (base_XFAIL = target_XFAIL = strstr (XFAIL, target->basename)) != NULL) &&
-		    target_XFAIL < strpbrk (XFAIL, " \t;:") &&
-		    target_XFAIL[-1] != '!')
-		{
-		    ctx.expectation = CAIRO_TEST_FAILURE;
-
-		    if (base_XFAIL != NULL) {
-			unsigned slen;
-
-			slen = strpbrk (base_XFAIL, " \t;:,") - base_XFAIL;
-			if (slen != strlen (target->basename))
-			    ctx.expectation = CAIRO_TEST_SUCCESS;
-		    }
-		}
-		else
-		    ctx.expectation = CAIRO_TEST_SUCCESS;
-	    } else
-		ctx.expectation = expectation;
-
 	    has_similar = runner.full_test ?
 			  cairo_test_target_has_similar (&ctx, target) :
 		          0;
@@ -800,7 +739,11 @@ main (int argc, char **argv)
 		    case CAIRO_TEST_SUCCESS:
 			target_skipped = FALSE;
 			break;
+		    case CAIRO_TEST_XFAILURE:
+			target_xfailed = TRUE;
+			break;
 		    case CAIRO_TEST_NO_MEMORY:
+		    case CAIRO_TEST_NEW:
 		    case CAIRO_TEST_FAILURE:
 			target_failed = TRUE;
 			break;
@@ -820,26 +763,19 @@ main (int argc, char **argv)
 							      list->test);
 		crashed = TRUE;
 	    } else if (target_failed) {
-		if (ctx.expectation == CAIRO_TEST_SUCCESS) {
+		target_status[n] = CAIRO_TEST_FAILURE;
-		    target_status[n] = CAIRO_TEST_FAILURE;
+		runner.num_failed_per_target[n]++;
-		    runner.num_failed_per_target[n]++;
+		runner.fails_per_target[n] = _list_prepend (runner.fails_per_target[n],
-		    runner.fails_per_target[n] = _list_prepend (runner.fails_per_target[n],
+							    list->test);
-								list->test);
-		} else
-		    target_status[n] = CAIRO_TEST_SUCCESS;
 
 		failed = TRUE;
+	    } else if (target_xfailed) {
+		target_status[n] = CAIRO_TEST_XFAILURE;
+		xfailed = TRUE;
 	    } else if (target_skipped) {
 		target_status[n] = CAIRO_TEST_UNTESTED;
 	    } else {
 		target_status[n] = CAIRO_TEST_SUCCESS;
-		/* An unexpected pass should also be flagged */
-		if (ctx.expectation != CAIRO_TEST_SUCCESS) {
-		    target_status[n] = CAIRO_TEST_FAILURE;
-		    runner.num_xpassed_per_target[n]++;
-		    runner.xpasses_per_target[n] = _list_prepend (runner.xpasses_per_target[n],
-								  list->test);
-		}
 		skipped = FALSE;
 	    }
 	}
@@ -866,44 +802,32 @@ main (int argc, char **argv)
 	    runner.num_crashed++;
 	    runner.passed = FALSE;
 	} else if (failed) {
-	    if (expectation == CAIRO_TEST_SUCCESS) {
+	    len = 0;
-		len = 0;
+	    for (n = 0 ; n < runner.base.num_targets; n++) {
-		for (n = 0 ; n < runner.base.num_targets; n++) {
+		if (target_status[n] == CAIRO_TEST_FAILURE) {
-		    if (target_status[n] == CAIRO_TEST_FAILURE) {
+		    if (strstr (targets,
-			if (strstr (targets,
+				runner.base.targets_to_test[n]->name) == NULL)
-				    runner.base.targets_to_test[n]->name) == NULL)
+		    {
-			{
+			len += snprintf (targets + len,
-			    len += snprintf (targets + len,
+					 sizeof (targets) - len,
-					     sizeof (targets) - len,
+					 "%s, ",
-					     "%s, ",
+					 runner.base.targets_to_test[n]->name);
-					     runner.base.targets_to_test[n]->name);
-			}
 		    }
 		}
-		if (len == 0) {
-		    _log (&runner.base, "%s: XFAIL\n", name);
-		    runner.num_xfailed++;
-		} else {
-		    targets[len-2] = '\0';
-		    _log (&runner.base, "%s: FAIL (%s)\n", name, targets);
-		    runner.num_failed++;
-		    runner.passed = FALSE;
-		}
-	    } else {
-		_log (&runner.base, "%s: XFAIL\n", name);
-		runner.num_xfailed++;
 	    }
+	    targets[len-2] = '\0';
+	    _log (&runner.base, "%s: FAIL (%s)\n", name, targets);
+	    runner.num_failed++;
+	    runner.passed = FALSE;
+	} else if (xfailed) {
+	    _log (&runner.base, "%s: XFAIL\n", name);
+	    runner.num_xfailed++;
 	} else if (skipped) {
 	    _log (&runner.base, "%s: UNTESTED\n", name);
 	    runner.num_skipped++;
 	} else {
-	    if (expectation == CAIRO_TEST_SUCCESS) {
+	    _log (&runner.base, "%s: PASS\n", name);
-		_log (&runner.base, "%s: PASS\n", name);
+	    runner.num_passed++;
-		runner.num_passed++;
-	    } else {
-		_log (&runner.base, "%s: XPASS\n", name);
-		runner.num_xpassed++;
-	    }
 	}
 	fflush (runner.base.log_file);
 
@@ -921,7 +845,6 @@ main (int argc, char **argv)
     for (n = 0 ; n < runner.base.num_targets; n++) {
 	runner.crashes_per_target[n] = _list_reverse (runner.crashes_per_target[n]);
 	runner.fails_per_target[n] = _list_reverse (runner.fails_per_target[n]);
-	runner.xpasses_per_target[n] = _list_reverse (runner.xpasses_per_target[n]);
     }
 
     _runner_print_results (&runner);

test/cairo-test.c

@@ -86,6 +86,10 @@
 #define TRUE !FALSE
 #endif
 
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(A) (sizeof(A) / sizeof (A[0]))
+#endif
+
 static const cairo_user_data_key_t _cairo_test_context_key;
 
 static void
@@ -95,12 +99,6 @@ static const char *fail_face = "", *xfail_face="", *normal_face = "";
 static cairo_bool_t print_fail_on_stdout;
 static int cairo_test_timeout = 60;
 
-#define CAIRO_TEST_LOG_SUFFIX ".log"
-#define CAIRO_TEST_PNG_SUFFIX ".out.png"
-#define CAIRO_TEST_REF_SUFFIX ".ref.png"
-#define CAIRO_TEST_DIFF_SUFFIX ".diff.png"
-#define CAIRO_TEST_OUTPUT_DIR "output"
-
 #define NUM_DEVICE_OFFSETS 2
 
 static char *
@@ -127,8 +125,7 @@ static void
 _cairo_test_init (cairo_test_context_t *ctx,
 		  const cairo_test_context_t *parent,
 		  const cairo_test_t *test,
-		  const char *test_name,
+		  const char *test_name)
-		  cairo_test_status_t expectation)
 {
     char *log_name;
 
@@ -140,7 +137,6 @@ _cairo_test_init (cairo_test_context_t *ctx,
 
     ctx->test = test;
     ctx->test_name = _cairo_test_fixup_name (test_name);
-    ctx->expectation = expectation;
 
     ctx->malloc_failure = 0;
 #if HAVE_MEMFAULT
@@ -212,14 +208,14 @@ _cairo_test_context_init_for_test (cairo_test_context_t *ctx,
 				   const cairo_test_context_t *parent,
 				   const cairo_test_t *test)
 {
-    _cairo_test_init (ctx, parent, test, test->name, CAIRO_TEST_SUCCESS);
+    _cairo_test_init (ctx, parent, test, test->name);
 }
 
 void
 cairo_test_init (cairo_test_context_t *ctx,
 		 const char *test_name)
 {
-    _cairo_test_init (ctx, NULL, NULL, test_name, CAIRO_TEST_SUCCESS);
+    _cairo_test_init (ctx, NULL, NULL, test_name);
 }
 
 static void
@@ -329,21 +325,24 @@ _xunlink (const cairo_test_context_t *ctx, const char *pathname)
 }
 
 char *
-cairo_test_reference_image_filename (const cairo_test_context_t *ctx,
+cairo_test_reference_filename (const cairo_test_context_t *ctx,
-	                             const char *base_name,
+			       const char *base_name,
-				     const char *test_name,
+			       const char *test_name,
-				     const char *target_name,
+			       const char *target_name,
-				     const char *base_target_name,
+			       const char *base_target_name,
-				     const char *format)
+			       const char *format,
+			       const char *suffix,
+			       const char *extension)
 {
     char *ref_name = NULL;
 
     /* First look for a previous build for comparison. */
     if (ctx->refdir != NULL) {
-	xasprintf (&ref_name, "%s/%s%s",
+	xasprintf (&ref_name, "%s/%s%s%s",
 		   ctx->refdir,
 		   base_name,
-		   CAIRO_TEST_PNG_SUFFIX);
+		   suffix,
+		   extension);
 	if (access (ref_name, F_OK) != 0)
 	    free (ref_name);
 	else
@@ -351,66 +350,72 @@ cairo_test_reference_image_filename (const cairo_test_context_t *ctx,
     }
 
     /* Next look for a target/format-specific reference image. */
-    xasprintf (&ref_name, "%s/%s.%s.%s%s",
+    xasprintf (&ref_name, "%s/%s.%s.%s%s%s",
 	       ctx->srcdir,
 	       test_name,
 	       target_name,
 	       format,
-	       CAIRO_TEST_REF_SUFFIX);
+	       suffix,
+	       extension);
     if (access (ref_name, F_OK) != 0)
 	free (ref_name);
     else
 	goto done;
 
     /* Next, look for target-specific reference image. */
-    xasprintf (&ref_name, "%s/%s.%s%s",
+    xasprintf (&ref_name, "%s/%s.%s%s%s",
 	       ctx->srcdir,
 	       test_name,
 	       target_name,
-	       CAIRO_TEST_REF_SUFFIX);
+	       suffix,
+	       extension);
     if (access (ref_name, F_OK) != 0)
 	free (ref_name);
     else
 	goto done;
 
     /* Next look for a base/format-specific reference image. */
-    xasprintf (&ref_name, "%s/%s.%s.%s%s",
+    xasprintf (&ref_name, "%s/%s.%s.%s%s%s",
 	       ctx->srcdir,
 	       test_name,
 	       base_target_name,
 	       format,
-	       CAIRO_TEST_REF_SUFFIX);
+	       suffix,
+	       extension);
     if (access (ref_name, F_OK) != 0)
 	free (ref_name);
     else
 	goto done;
 
     /* Next, look for base-specific reference image. */
-    xasprintf (&ref_name, "%s/%s.%s%s",
+    xasprintf (&ref_name, "%s/%s.%s%s%s",
 	       ctx->srcdir,
 	       test_name,
 	       base_target_name,
-	       CAIRO_TEST_REF_SUFFIX);
+	       suffix,
+	       extension);
     if (access (ref_name, F_OK) != 0)
 	free (ref_name);
     else
 	goto done;
 
     /* Next, look for format-specific reference image. */
-    xasprintf (&ref_name, "%s/%s.%s%s",
+    xasprintf (&ref_name, "%s/%s.%s%s%s",
 	       ctx->srcdir,
 	       test_name,
 	       format,
-	       CAIRO_TEST_REF_SUFFIX);
+	       suffix,
+	       extension);
     if (access (ref_name, F_OK) != 0)
 	free (ref_name);
     else
 	goto done;
 
     /* Finally, look for the standard reference image. */
-    xasprintf (&ref_name, "%s/%s%s", ctx->srcdir,
+    xasprintf (&ref_name, "%s/%s%s%s", ctx->srcdir,
 	       test_name,
-	       CAIRO_TEST_REF_SUFFIX);
+	       suffix,
+	       extension);
     if (access (ref_name, F_OK) != 0)
 	free (ref_name);
     else
@@ -546,22 +551,31 @@ cairo_test_get_reference_image (cairo_test_context_t *ctx,
 
 static cairo_bool_t
 cairo_test_file_is_older (const char *filename,
-	                  const char *ref_filename)
+	                  char **ref_filenames,
+			  int num_ref_filenames)
 {
 #if HAVE_SYS_STAT_H
-    struct stat st, ref;
+    struct stat st;
 
     if (stat (filename, &st) < 0)
 	return FALSE;
 
-    if (stat (ref_filename, &ref) < 0)
+    while (num_ref_filenames--) {
-	return TRUE;
+	struct stat ref;
+	char *ref_filename = ref_filenames++;
 
-    return st.st_mtime < ref.st_mtime;
+	if (ref_filename == NULL)
-#else
+	    continue;
-    /* XXX */
+
-    return FALSE;
+	if (stat (ref_filename++, &ref) < 0)
+	    continue;
+
+	if (st.st_mtime <= ref.st_mtime)
+	    return TRUE;
+    }
 #endif
+
+    return FALSE;
 }
 
 static cairo_bool_t
@@ -571,6 +585,9 @@ cairo_test_files_equal (const char *test_filename,
     FILE *test, *pass;
     int t, p;
 
+    if (test_filename == NULL || pass_filename == NULL)
+	return FALSE;
+
     test = fopen (test_filename, "rb");
     if (test == NULL)
 	return FALSE;
@@ -659,7 +676,11 @@ cairo_test_for_target (cairo_test_context_t		 *ctx,
     const char *empty_str = "";
     char *offset_str, *thread_str;
     char *base_name, *base_path;
-    char *png_path, *ref_path, *diff_path;
+    char *out_png_path;
+    char *ref_path = NULL, *ref_png_path;
+    char *new_path = NULL, *new_png_path;
+    char *xfail_path = NULL, *xfail_png_path;
+    char *diff_png_path;
     char *test_filename = NULL, *pass_filename = NULL, *fail_filename = NULL;
     cairo_test_status_t ret;
     cairo_content_t expected_content;
@@ -700,18 +721,64 @@ cairo_test_for_target (cairo_test_context_t		 *ctx,
       free (thread_str);
 
 
-    ref_path = cairo_test_reference_image_filename (ctx,
+    ref_png_path = cairo_test_reference_filename (ctx,
+						  base_name,
+						  ctx->test_name,
+						  target->name,
+						  target->basename,
+						  format,
+						  CAIRO_TEST_REF_SUFFIX,
+						  CAIRO_TEST_PNG_EXTENSION);
+    new_png_path = cairo_test_reference_filename (ctx,
+						  base_name,
+						  ctx->test_name,
+						  target->name,
+						  target->basename,
+						  format,
+						  CAIRO_TEST_NEW_SUFFIX,
+						  CAIRO_TEST_PNG_EXTENSION);
+    xfail_png_path = cairo_test_reference_filename (ctx,
 						    base_name,
 						    ctx->test_name,
 						    target->name,
 						    target->basename,
-						    format);
+						    format,
+						    CAIRO_TEST_XFAIL_SUFFIX,
+						    CAIRO_TEST_PNG_EXTENSION);
+
+    if (target->file_extension != NULL) {
+	ref_path = cairo_test_reference_filename (ctx,
+						  base_name,
+						  ctx->test_name,
+						  target->name,
+						  target->basename,
+						  format,
+						  CAIRO_TEST_REF_SUFFIX,
+						  target->file_extension);
+	new_path = cairo_test_reference_filename (ctx,
+						  base_name,
+						  ctx->test_name,
+						  target->name,
+						  target->basename,
+						  format,
+						  CAIRO_TEST_NEW_SUFFIX,
+						  target->file_extension);
+	xfail_path = cairo_test_reference_filename (ctx,
+						    base_name,
+						    ctx->test_name,
+						    target->name,
+						    target->basename,
+						    format,
+						    CAIRO_TEST_XFAIL_SUFFIX,
+						    target->file_extension);
+    }
+
     have_output_dir = _cairo_test_mkdir (CAIRO_TEST_OUTPUT_DIR);
     xasprintf (&base_path, "%s/%s",
 	       have_output_dir ? CAIRO_TEST_OUTPUT_DIR : ".",
 	       base_name);
-    xasprintf (&png_path, "%s" CAIRO_TEST_PNG_SUFFIX, base_path);
+    xasprintf (&out_png_path, "%s" CAIRO_TEST_OUT_PNG, base_path);
-    xasprintf (&diff_path, "%s" CAIRO_TEST_DIFF_SUFFIX, base_path);
+    xasprintf (&diff_png_path, "%s" CAIRO_TEST_DIFF_PNG, base_path);
 
     if (ctx->test->requirements != NULL) {
 	const char *required;
@@ -973,16 +1040,16 @@ REPEAT:
 	    }
 	}
 
-	if (ref_path == NULL) {
+	if (ref_png_path == NULL) {
 	    cairo_test_log (ctx, "Error: Cannot find reference image for %s\n",
 			    base_name);
 
 	    /* we may be running this test to generate reference images */
-	    _xunlink (ctx, png_path);
+	    _xunlink (ctx, out_png_path);
 	    test_image = target->get_image_surface (surface, 0,
 		                                    ctx->test->width,
 						    ctx->test->height);
-	    diff_status = cairo_surface_write_to_png (test_image, png_path);
+	    diff_status = cairo_surface_write_to_png (test_image, out_png_path);
 	    if (diff_status) {
 		cairo_test_log (ctx,
 			        "Error: Failed to write output image: %s\n",
@@ -991,11 +1058,20 @@ REPEAT:
 	    have_output = TRUE;
 	    cairo_surface_destroy (test_image);
 
-	    ret = CAIRO_TEST_FAILURE;
+	    ret = CAIRO_TEST_XFAILURE;
 	    goto UNWIND_CAIRO;
 	}
 
 	if (target->file_extension != NULL) { /* compare vector surfaces */
+	    char *filenames[] = {
+		ref_png_path,
+		ref_path,
+		new_png_path,
+		new_path,
+		xfail_png_path,
+		xfail_path,
+	    };
+
 	    xasprintf (&test_filename, "%s.out%s",
 		       base_path, target->file_extension);
 	    xasprintf (&pass_filename, "%s.pass%s",
@@ -1003,10 +1079,37 @@ REPEAT:
 	    xasprintf (&fail_filename, "%s.fail%s",
 		       base_path, target->file_extension);
 
-	    if (cairo_test_file_is_older (pass_filename, ref_path))
+	    if (cairo_test_file_is_older (pass_filename,
+					  filenames,
+					  ARRAY_SIZE (filenames)))
+	    {
 		_xunlink (ctx, pass_filename);
-	    if (cairo_test_file_is_older (fail_filename, ref_path))
+	    }
+	    if (cairo_test_file_is_older (fail_filename,
+					  filenames,
+					  ARRAY_SIZE (filenames)))
+	    {
 		_xunlink (ctx, fail_filename);
+	    }
+
+	    if (cairo_test_files_equal (out_png_path, ref_path)) {
+		cairo_test_log (ctx, "Vector surface matches reference.\n");
+		have_output = FALSE;
+		ret = CAIRO_TEST_SUCCESS;
+		goto UNWIND_CAIRO;
+	    }
+	    if (cairo_test_files_equal (out_png_path, new_path)) {
+		cairo_test_log (ctx, "Vector surface matches current failure.\n");
+		have_output = FALSE;
+		ret = CAIRO_TEST_NEW;
+		goto UNWIND_CAIRO;
+	    }
+	    if (cairo_test_files_equal (out_png_path, xfail_path)) {
+		cairo_test_log (ctx, "Vector surface matches known failure.\n");
+		have_output = FALSE;
+		ret = CAIRO_TEST_XFAILURE;
+		goto UNWIND_CAIRO;
+	    }
 
 	    if (cairo_test_files_equal (test_filename, pass_filename)) {
 		/* identical output as last known PASS */
@@ -1036,8 +1139,8 @@ REPEAT:
 	    goto UNWIND_CAIRO;
 	}
 
-	_xunlink (ctx, png_path);
+	_xunlink (ctx, out_png_path);
-	diff_status = cairo_surface_write_to_png (test_image, png_path);
+	diff_status = cairo_surface_write_to_png (test_image, out_png_path);
 	if (diff_status) {
 	    cairo_test_log (ctx, "Error: Failed to write output image: %s\n",
 			    cairo_status_to_string (diff_status));
@@ -1049,47 +1152,86 @@ REPEAT:
 
 	/* binary compare png files (no decompression) */
 	if (target->file_extension == NULL) {
-	    xasprintf (&test_filename, "%s", png_path);
+	    char *filenames[] = {
+		ref_png_path,
+		new_png_path,
+		xfail_png_path,
+	    };
+
+	    xasprintf (&test_filename, "%s", out_png_path);
 	    xasprintf (&pass_filename, "%s.pass.png", base_path);
 	    xasprintf (&fail_filename, "%s.fail.png", base_path);
 
+	    if (cairo_test_file_is_older (pass_filename,
+					  filenames,
+					  ARRAY_SIZE (filenames)))
+	    {
+		_xunlink (ctx, pass_filename);
+	    }
+	    if (cairo_test_file_is_older (fail_filename,
+					  filenames,
+					  ARRAY_SIZE (filenames)))
+	    {
+		_xunlink (ctx, fail_filename);
+	    }
+
 	    if (cairo_test_files_equal (test_filename, pass_filename)) {
-		/* identical output as last known PASS, pass */
 		cairo_test_log (ctx, "PNG file exactly matches last pass.\n");
 		cairo_surface_destroy (test_image);
 		ret = CAIRO_TEST_SUCCESS;
 		goto UNWIND_CAIRO;
 	    }
-	    if (cairo_test_files_equal (png_path, ref_path)) {
+	    if (cairo_test_files_equal (out_png_path, ref_png_path)) {
-		/* identical output as reference image */
 		cairo_test_log (ctx, "PNG file exactly reference image.\n");
 		cairo_surface_destroy (test_image);
 		ret = CAIRO_TEST_SUCCESS;
 		goto UNWIND_CAIRO;
 	    }
-
+	    if (cairo_test_files_equal (out_png_path, new_png_path)) {
+		cairo_test_log (ctx, "PNG file exactly current failure image.\n");
+		cairo_surface_destroy (test_image);
+		ret = CAIRO_TEST_NEW;
+		goto UNWIND_CAIRO;
+	    }
+	    if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
+		cairo_test_log (ctx, "PNG file exactly known failure image.\n");
+		cairo_surface_destroy (test_image);
+		ret = CAIRO_TEST_XFAILURE;
+		goto UNWIND_CAIRO;
+	    }
 	    if (cairo_test_files_equal (test_filename, fail_filename)) {
 		cairo_test_log (ctx, "PNG file exactly matches last fail.\n");
-		/* identical output as last known FAIL, fail */
 		have_result = TRUE; /* presume these were kept around as well */
 		cairo_surface_destroy (test_image);
 		ret = CAIRO_TEST_FAILURE;
 		goto UNWIND_CAIRO;
 	    }
 	} else {
-	    if (cairo_test_files_equal (png_path, ref_path)) {
+	    if (cairo_test_files_equal (out_png_path, ref_png_path)) {
 		cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
 		cairo_surface_destroy (test_image);
 		ret = CAIRO_TEST_SUCCESS;
 		goto UNWIND_CAIRO;
 	    }
+	    if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
+		cairo_test_log (ctx, "PNG file exactly matches current failure image.\n");
+		cairo_surface_destroy (test_image);
+		ret = CAIRO_TEST_NEW;
+		goto UNWIND_CAIRO;
+	    }
+	    if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
+		cairo_test_log (ctx, "PNG file exactly matches known failure image.\n");
+		cairo_surface_destroy (test_image);
+		ret = CAIRO_TEST_XFAILURE;
+		goto UNWIND_CAIRO;
+	    }
 	}
 
-	ref_image = cairo_test_get_reference_image (ctx, ref_path,
+	ref_image = cairo_test_get_reference_image (ctx, ref_png_path,
 						    target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
 	if (cairo_surface_status (ref_image)) {
 	    cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
-			    ref_path,
+			    ref_png_path,
 			    cairo_status_to_string (cairo_surface_status (ref_image)));
 	    cairo_surface_destroy (ref_image);
 	    cairo_surface_destroy (test_image);
@@ -1105,7 +1247,7 @@ REPEAT:
 				  test_image, ref_image,
 				  diff_image,
 				  &result);
-	_xunlink (ctx, diff_path);
+	_xunlink (ctx, diff_png_path);
 	if (diff_status) {
 	    cairo_test_log (ctx, "Error: Failed to compare images: %s\n",
 			    cairo_status_to_string (diff_status));
@@ -1116,7 +1258,7 @@ REPEAT:
 	{
 	    ret = CAIRO_TEST_FAILURE;
 
-	    diff_status = cairo_surface_write_to_png (diff_image, diff_path);
+	    diff_status = cairo_surface_write_to_png (diff_image, diff_png_path);
 	    if (diff_status) {
 		cairo_test_log (ctx, "Error: Failed to write differences image: %s\n",
 				cairo_status_to_string (diff_status));
@@ -1147,7 +1289,7 @@ UNWIND_CAIRO:
     }
 
 #if HAVE_MEMFAULT
-    if (ret == CAIRO_TEST_FAILURE && ctx->expectation != CAIRO_TEST_FAILURE)
+    if (ret == CAIRO_TEST_FAILURE)
 	MEMFAULT_PRINT_FAULTS ();
 #endif
     cairo_destroy (cr);
@@ -1166,11 +1308,8 @@ UNWIND_SURFACE:
 #endif
 
 	if (MEMFAULT_COUNT_LEAKS () > 0) {
-	    if (ret != CAIRO_TEST_FAILURE ||
+	    if (ret != CAIRO_TEST_FAILURE)
-		ctx->expectation == CAIRO_TEST_FAILURE)
-	    {
 		MEMFAULT_PRINT_FAULTS ();
-	    }
 	    MEMFAULT_PRINT_LEAKS ();
 	}
     }
@@ -1181,22 +1320,32 @@ UNWIND_SURFACE:
 
     if (ctx->thread == 0) {
 	if (have_output)
-	    cairo_test_log (ctx, "OUTPUT: %s\n", png_path);
+	    cairo_test_log (ctx, "OUTPUT: %s\n", out_png_path);
 
 	if (have_result) {
 	    cairo_test_log (ctx,
 		            "REFERENCE: %s\nDIFFERENCE: %s\n",
-			    ref_path, diff_path);
+			    ref_png_path, diff_png_path);
 	}
     }
 
 UNWIND_STRINGS:
-    if (png_path)
+    if (out_png_path)
-      free (png_path);
+      free (out_png_path);
+    if (ref_png_path)
+      free (ref_png_path);
     if (ref_path)
       free (ref_path);
-    if (diff_path)
+    if (new_png_path)
-      free (diff_path);
+      free (new_png_path);
+    if (new_path)
+      free (new_path);
+    if (xfail_png_path)
+      free (xfail_png_path);
+    if (xfail_path)
+      free (xfail_path);
+    if (diff_png_path)
+      free (diff_png_path);
     if (base_path)
       free (base_path);
     if (base_name)
@@ -1303,10 +1452,13 @@ _cairo_test_context_run_for_target (cairo_test_context_t *ctx,
 	    printf ("PASS\n");
 	    cairo_test_log (ctx, "PASS\n");
 	    break;
+
 	case CAIRO_TEST_UNTESTED:
 	    printf ("UNTESTED\n");
 	    cairo_test_log (ctx, "UNTESTED\n");
 	    break;
+
+	default:
 	case CAIRO_TEST_CRASHED:
 	    if (print_fail_on_stdout && ctx->thread == 0) {
 		printf ("!!!CRASHED!!!\n");
@@ -1321,36 +1473,51 @@ _cairo_test_context_run_for_target (cairo_test_context_t *ctx,
 		     cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
 		     fail_face, normal_face);
 	    break;
-	default:
+
+	case CAIRO_TEST_XFAILURE:
+	    if (print_fail_on_stdout && ctx->thread == 0) {
+		printf ("XFAIL\n");
+	    } else {
+		/* eat the test name */
+		printf ("\r");
+		fflush (stdout);
+	    }
+	    fprintf (stderr, "%s.%s.%s [%d]%s:\t%sXFAIL%s\n",
+		     ctx->test_name, target->name,
+		     cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
+		     xfail_face, normal_face);
+	    cairo_test_log (ctx, "XFAIL\n");
+	    break;
+
+	case CAIRO_TEST_NEW:
+	    if (print_fail_on_stdout && ctx->thread == 0) {
+		printf ("NEW\n");
+	    } else {
+		/* eat the test name */
+		printf ("\r");
+		fflush (stdout);
+	    }
+	    fprintf (stderr, "%s.%s.%s [%d]%s:\t%sNEW%s\n",
+		     ctx->test_name, target->name,
+		     cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
+		     fail_face, normal_face);
+	    cairo_test_log (ctx, "NEW\n");
+	    break;
+
 	case CAIRO_TEST_NO_MEMORY:
 	case CAIRO_TEST_FAILURE:
-	    if (ctx->expectation == CAIRO_TEST_FAILURE) {
+	    if (print_fail_on_stdout && ctx->thread == 0) {
-		if (print_fail_on_stdout && ctx->thread == 0) {
+		printf ("FAIL\n");
-		    printf ("XFAIL\n");
-		} else {
-		    /* eat the test name */
-		    printf ("\r");
-		    fflush (stdout);
-		}
-		fprintf (stderr, "%s.%s.%s [%d]%s:\t%sXFAIL%s\n",
-			 ctx->test_name, target->name,
-			 cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
-			 xfail_face, normal_face);
-		cairo_test_log (ctx, "XFAIL\n");
 	    } else {
-		if (print_fail_on_stdout && ctx->thread == 0) {
+		/* eat the test name */
-		    printf ("FAIL\n");
+		printf ("\r");
-		} else {
+		fflush (stdout);
-		    /* eat the test name */
-		    printf ("\r");
-		    fflush (stdout);
-		}
-		fprintf (stderr, "%s.%s.%s [%d]%s:\t%sFAIL%s\n",
-			 ctx->test_name, target->name,
-			 cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
-			 fail_face, normal_face);
-		cairo_test_log (ctx, "FAIL\n");
 	    }
+	    fprintf (stderr, "%s.%s.%s [%d]%s:\t%sFAIL%s\n",
+		     ctx->test_name, target->name,
+		     cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
+		     fail_face, normal_face);
+	    cairo_test_log (ctx, "FAIL\n");
 	    break;
 	}
 	fflush (stdout);
@@ -1370,17 +1537,19 @@ _cairo_test_context_run_for_target (cairo_test_context_t *ctx,
 	case CAIRO_TEST_UNTESTED:
 	    printf ("UNTESTED\n");
 	    break;
+	default:
 	case CAIRO_TEST_CRASHED:
 	    printf ("!!!CRASHED!!!\n");
 	    break;
-	default:
+	case CAIRO_TEST_XFAILURE:
+	    printf ("XFAIL\n");
+	    break;
+	case CAIRO_TEST_NEW:
+	    printf ("NEW\n");
+	    break;
 	case CAIRO_TEST_NO_MEMORY:
 	case CAIRO_TEST_FAILURE:
-	    if (ctx->expectation == CAIRO_TEST_FAILURE) {
+	    printf ("FAIL\n");
-		printf ("XFAIL\n");
-	    } else {
-		printf ("FAIL\n");
-	    }
 	    break;
 	}
 
@@ -1473,19 +1642,15 @@ cairo_test_run_threaded (void *closure)
 
 
 static cairo_test_status_t
-cairo_test_expecting (const cairo_test_t *test,
+cairo_test_expecting (const cairo_test_t *test)
-		      cairo_test_status_t expectation)
 {
     cairo_test_context_t ctx;
     cairo_test_status_t ret = CAIRO_TEST_SUCCESS;
     size_t num_threads;
 
-    _cairo_test_init (&ctx, NULL, test, test->name, expectation);
+    _cairo_test_init (&ctx, NULL, test, test->name);
     printf ("%s\n", test->description);
 
-    if (expectation == CAIRO_TEST_FAILURE)
-	printf ("Expecting failure\n");
-
 #if HAVE_PTHREAD_H
     num_threads = 0;
     if (getenv ("CAIRO_TEST_NUM_THREADS"))
@@ -1517,20 +1682,6 @@ cairo_test_expecting (const cairo_test_t *test,
     if (ret != CAIRO_TEST_SUCCESS)
         printf ("Check %s%s out for more information.\n", ctx.test_name, CAIRO_TEST_LOG_SUFFIX);
 
-    /* if the set of targets to test was limited using CAIRO_TEST_TARGET, we
-     * behave slightly differently, to ensure that limiting the targets does
-     * not increase the number of tests failing. */
-    if (ctx.limited_targets) {
-	/* if all passed, but expecting failure, return failure to not
-	 * trigger an XPASS failure */
-	if (expectation == CAIRO_TEST_FAILURE && ret == CAIRO_TEST_SUCCESS) {
-	    printf ("All tested backends passed, but tested targets are manually limited\n"
-		    "and the test suite expects this test to fail for at least one target.\n"
-		    "Intentionally failing the test, to not fail the suite.\n");
-	    ret = CAIRO_TEST_FAILURE;
-	}
-    }
-
     cairo_test_fini (&ctx);
 
     return ret;
@@ -1545,7 +1696,7 @@ cairo_test (const cairo_test_t *test)
     _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR);
 #endif
 
-    return cairo_test_expecting (test, CAIRO_TEST_SUCCESS);
+    return cairo_test_expecting (test);
 }
 
 const cairo_test_context_t *

test/cairo-test.h

@@ -79,10 +79,51 @@ cairo_test_NaN (void)
 #endif
 }
 
+#define CAIRO_TEST_OUTPUT_DIR "output"
+
+#define CAIRO_TEST_LOG_SUFFIX ".log"
+
+/* What is a fail and what isn't?
+ * When running the test suite we want to detect unexpected output. This
+ * can be caused by a change we have made to cairo itself, or a change
+ * in our environment. To capture this we classify the expected output into 3
+ * classes:
+ *
+ *   REF  -- Perfect output.
+ *           Might be different for each backend, due to slight implementation
+ *           differences.
+ *
+ *   NEW  -- A new failure. We have uncovered a bug within cairo and have
+ *           recorded the current failure (along with the expected output
+ *           if possible!) so we can detect any changes in our attempt to
+ *           fix the bug.
+ *
+ *  XFAIL -- An external failure. We believe the cairo output is perfect,
+ *           but an external renderer is causing gross failure.
+ *           (We also use this to capture current WONTFIX issues within cairo,
+ *           such as overflow in internal coordinates, so as not to distract
+ *           us when regression testing.)
+ *
+ *  If no REF is given for a test, then it is assumed to be XFAIL.
+ */
+#define CAIRO_TEST_REF_SUFFIX ".ref"
+#define CAIRO_TEST_XFAIL_SUFFIX ".xfail"
+#define CAIRO_TEST_NEW_SUFFIX ".new"
+
+#define CAIRO_TEST_OUT_SUFFIX ".out"
+#define CAIRO_TEST_DIFF_SUFFIX ".diff"
+
+#define CAIRO_TEST_PNG_EXTENSION ".png"
+#define CAIRO_TEST_OUT_PNG CAIRO_TEST_OUT_SUFFIX CAIRO_TEST_PNG_EXTENSION
+#define CAIRO_TEST_REF_PNG CAIRO_TEST_REF_SUFFIX CAIRO_TEST_PNG_EXTENSION
+#define CAIRO_TEST_DIFF_PNG CAIRO_TEST_DIFF_SUFFIX CAIRO_TEST_PNG_EXTENSION
+
 typedef enum cairo_test_status {
     CAIRO_TEST_SUCCESS = 0,
     CAIRO_TEST_NO_MEMORY,
     CAIRO_TEST_FAILURE,
+    CAIRO_TEST_NEW,
+    CAIRO_TEST_XFAILURE,
     CAIRO_TEST_CRASHED,
     CAIRO_TEST_UNTESTED = 77 /* match automake's skipped exit status */
 } cairo_test_status_t;
@@ -161,7 +202,6 @@ cairo_test_register (const cairo_test_t *test);
 struct _cairo_test_context {
     const cairo_test_t *test;
     const char *test_name;
-    cairo_test_status_t expectation;
 
     FILE *log_file;
     const char *srcdir; /* directory containing sources and input data */
@@ -234,12 +274,14 @@ cairo_test_status_from_status (const cairo_test_context_t *ctx,
 			       cairo_status_t status);
 
 char *
-cairo_test_reference_image_filename (const cairo_test_context_t *ctx,
+cairo_test_reference_filename (const cairo_test_context_t *ctx,
-	                             const char *base_name,
+			       const char *base_name,
-				     const char *test_name,
+			       const char *test_name,
-				     const char *target_name,
+			       const char *target_name,
-				     const char *base_target_name,
+			       const char *base_target_name,
-				     const char *format);
+			       const char *format,
+			       const char *suffix,
+			       const char *extension);
 
 cairo_surface_t *
 cairo_test_get_reference_image (cairo_test_context_t *ctx,

test/clip-operator.c

@@ -182,7 +182,7 @@ draw (cairo_t *cr, int width, int height)
 
 CAIRO_TEST (clip_operator,
 	    "Surface clipping with different operators",
-	    "XFAIL=svg12 clip", /* keywords */
+	    "clip", /* keywords */
 	    NULL, /* requirements */
 	    IMAGE_WIDTH, IMAGE_HEIGHT,
 	    NULL, draw)

test/composite-integer-translate-source.c

@@ -58,7 +58,7 @@ draw (cairo_t *cr, int width, int height)
 
 CAIRO_TEST (composite_integer_translate_source,
 	    "Test simple compositing: integer-translation 32->32 SOURCE",
-	    "XFAIL=svg12 composite", /* keywords */
+	    "composite", /* keywords */
 	    NULL, /* requirements */
 	    SIZE, SIZE,
 	    NULL, draw)

test/degenerate-dash.c

@@ -83,7 +83,7 @@ draw (cairo_t *cr, int width, int height)
 
 CAIRO_TEST (degenerate_dash,
 	    "Tests the behaviour of dashed segments that end on a off-on transition",
-	    "XFAIL=ps dash, degenerate", /* keywords */
+	    "dash, degenerate", /* keywords */
 	    NULL, /* requirementts */
 	    210 + 2*PAD, 210 + 2*PAD,
 	    NULL, draw)

test/degenerate-path.c

@@ -112,7 +112,7 @@ draw (cairo_t *cr, int width, int height)
 
 CAIRO_TEST (degenerate_path,
 	    "Tests the behaviour of degenerate paths with different cap types",
-	    "XFAIL=ps degenerate", /* keywords */
+	    "degenerate", /* keywords */
 	    NULL, /* requirements */
 	    3*(PAD+LINE_WIDTH+PAD), 8*(LINE_WIDTH+PAD) + PAD,
 	    NULL, draw)

test/device-offset-scale.c

@@ -67,7 +67,7 @@ draw (cairo_t *cr, int width, int height)
 
 CAIRO_TEST (device_offset_scale,
 	    "Test that the device-offset transform is transformed by the ctm.",
-	    "XFAIL=svg device-offset", /* keywords */
+	    "device-offset", /* keywords */
 	    NULL, /* requirements */
 	    WIDTH, HEIGHT,
 	    NULL, draw)

test/extend-pad-border.c

@@ -89,7 +89,7 @@ draw (cairo_t *cr, int width, int height)
 
 CAIRO_TEST (extend_pad_border,
 	    "Test CAIRO_EXTEND_PAD for surface patterns",
-	    "XFAIL=!image,pdf,ps,svg extend", /* keywords */
+	    "extend", /* keywords */
 	    NULL, /* requirements */
 	    SIZE, SIZE,
 	    NULL, draw)

test/fallback-resolution.c

@@ -190,12 +190,14 @@ check_result (cairo_test_context_t *ctx,
     }
 
     format = cairo_boilerplate_content_name (target->content);
-    ref_name = cairo_test_reference_image_filename (ctx,
+    ref_name = cairo_test_reference_filename (ctx,
-	                                            base_name,
+					      base_name,
-						    test_name,
+					      test_name,
-						    target->name,
+					      target->name,
-						    target->basename,
+					      target->basename,
-						    format);
+					      format,
+					      CAIRO_TEST_REF_SUFFIX,
+					      CAIRO_TEST_PNG_EXTENSION);
     if (ref_name == NULL) {
 	cairo_test_log (ctx, "Error: Cannot find reference image for %s\n",
 		        base_name);

test/fill-and-stroke-alpha-add.c

@@ -101,7 +101,7 @@ draw (cairo_t *cr, int width, int height)
 
 CAIRO_TEST (fill_and_stroke_alpha_add,
 	    "Use a group to fill/stroke a path (each with different alpha) using DEST_OUT and ADD to combine",
-	    "XFAIL=svg12 fill-and-stroke, fill, stroke", /* keywords */
+	    "fill-and-stroke, fill, stroke", /* keywords */
 	    NULL, /* requirements */
 	    2 * SIZE + 4 * PAD, SIZE + 2 * PAD,
 	    NULL, draw)

test/fill-degenerate-sort-order.c

@@ -71,7 +71,7 @@ draw (cairo_t *cr, int width, int height)
 
 CAIRO_TEST (fill_degenerate_sort_order,
 	    "Tests the tessellator's event comparator with degenerate input",
-	    "XFAIL=ps degenerate, fill", /* keywords */
+	    "degenerate, fill", /* keywords */
 	    NULL, /* requirements */
 	    190, 120,
 	    NULL, draw)

test/finer-grained-fallbacks.c

@@ -174,7 +174,7 @@ draw (cairo_t *cr, int width, int height)
 
 CAIRO_TEST (finer_grained_fallbacks,
 	    "Test that multiple PS/PDF fallback images in various locations are correct",
-	    "XFAIL=svg12 fallbacks", /* keywords */
+	    "fallbacks", /* keywords */
 	    NULL, /* requirements */
 	    WIDTH, HEIGHT,
 	    NULL, draw)