From bd3d9ef3d1dbc5364e79e6afb47d9e124cb61ca4 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 11 Jun 2010 09:12:16 +0100 Subject: [PATCH] test: Distinguish tests that throw an error from a normal fail. Hitting an error in a test case is almost as bad as crashing, and the severity may be lost amidst "normal" failures. So introduce a new class of ERROR so that we can immediately spot these during a test run, and appropriately log them afterwards. --- test/cairo-test-runner.c | 89 +++++++++++++++++++++++++++++++++++++++- test/cairo-test.c | 20 ++++++++- test/cairo-test.h | 1 + 3 files changed, 107 insertions(+), 3 deletions(-) diff --git a/test/cairo-test-runner.c b/test/cairo-test-runner.c index d130321f0..aa9e1013a 100644 --- a/test/cairo-test-runner.c +++ b/test/cairo-test-runner.c @@ -78,15 +78,19 @@ typedef struct _cairo_test_runner { int num_skipped; int num_failed; int num_xfailed; + int num_error; int num_crashed; cairo_test_list_t *crashes_preamble; + cairo_test_list_t *errors_preamble; cairo_test_list_t *fails_preamble; cairo_test_list_t **crashes_per_target; + cairo_test_list_t **errors_per_target; cairo_test_list_t **fails_per_target; int *num_failed_per_target; + int *num_error_per_target; int *num_crashed_per_target; cairo_bool_t foreground; @@ -384,13 +388,18 @@ _runner_init (cairo_test_runner_t *runner) runner->fails_preamble = NULL; runner->crashes_preamble = NULL; + runner->errors_preamble = NULL; runner->fails_per_target = xcalloc (sizeof (cairo_test_list_t *), runner->base.num_targets); runner->crashes_per_target = xcalloc (sizeof (cairo_test_list_t *), runner->base.num_targets); + runner->errors_per_target = xcalloc (sizeof (cairo_test_list_t *), + runner->base.num_targets); runner->num_failed_per_target = xcalloc (sizeof (int), runner->base.num_targets); + runner->num_error_per_target = xcalloc (sizeof (int), + runner->base.num_targets); runner->num_crashed_per_target = xcalloc (sizeof (int), runner->base.num_targets); } @@ -443,6 +452,21 @@ _runner_print_details (cairo_test_runner_t *runner) } _log (&runner->base, "\n"); } + if (runner->errors_preamble) { + int count = 0; + + for (list = runner->errors_preamble; list != NULL; list = list->next) + count++; + + _log (&runner->base, "Preamble: %d error -", count); + + for (list = runner->errors_preamble; list != NULL; list = list->next) { + char *name = cairo_test_get_name (list->test); + _log (&runner->base, " %s", name); + free (name); + } + _log (&runner->base, "\n"); + } if (runner->fails_preamble) { int count = 0; @@ -479,6 +503,22 @@ _runner_print_details (cairo_test_runner_t *runner) } _log (&runner->base, "\n"); } + if (runner->num_error_per_target[n]) { + _log (&runner->base, "%s (%s): %d error -", + target->name, + cairo_boilerplate_content_name (target->content), + runner->num_error_per_target[n]); + + for (list = runner->errors_per_target[n]; + list != NULL; + list = list->next) + { + char *name = cairo_test_get_name (list->test); + _log (&runner->base, " %s", name); + free (name); + } + _log (&runner->base, "\n"); + } if (runner->num_failed_per_target[n]) { _log (&runner->base, "%s (%s): %d failed -", @@ -519,16 +559,20 @@ _runner_fini (cairo_test_runner_t *runner) unsigned int n; _list_free (runner->crashes_preamble); + _list_free (runner->errors_preamble); _list_free (runner->fails_preamble); for (n = 0; n < runner->base.num_targets; n++) { _list_free (runner->crashes_per_target[n]); + _list_free (runner->errors_per_target[n]); _list_free (runner->fails_per_target[n]); } free (runner->crashes_per_target); + free (runner->errors_per_target); free (runner->fails_per_target); free (runner->num_crashed_per_target); + free (runner->num_error_per_target); free (runner->num_failed_per_target); cairo_test_fini (&runner->base); @@ -700,7 +744,7 @@ main (int argc, char **argv) for (test = tests; test != NULL; test = test->next) { cairo_test_context_t ctx; cairo_test_status_t status; - cairo_bool_t failed = FALSE, xfailed = FALSE, crashed = FALSE, skipped = TRUE; + cairo_bool_t failed = FALSE, xfailed = FALSE, error = FALSE, crashed = FALSE, skipped = TRUE; cairo_bool_t in_preamble = FALSE; char *name = cairo_test_get_name (test); int i; @@ -810,6 +854,13 @@ main (int argc, char **argv) failed = TRUE; goto TEST_DONE; + case CAIRO_TEST_ERROR: + runner.errors_preamble = _list_prepend (runner.errors_preamble, + test); + in_preamble = TRUE; + failed = TRUE; + goto TEST_DONE; + case CAIRO_TEST_NO_MEMORY: case CAIRO_TEST_CRASHED: runner.crashes_preamble = _list_prepend (runner.crashes_preamble, @@ -830,6 +881,7 @@ main (int argc, char **argv) const cairo_boilerplate_target_t *target; cairo_bool_t target_failed = FALSE, target_xfailed = FALSE, + target_error = FALSE, target_crashed = FALSE, target_skipped = TRUE; int has_similar; @@ -853,11 +905,14 @@ main (int argc, char **argv) case CAIRO_TEST_XFAILURE: target_xfailed = TRUE; break; - case CAIRO_TEST_NO_MEMORY: case CAIRO_TEST_NEW: case CAIRO_TEST_FAILURE: target_failed = TRUE; break; + case CAIRO_TEST_ERROR: + target_error = TRUE; + break; + case CAIRO_TEST_NO_MEMORY: case CAIRO_TEST_CRASHED: target_crashed = TRUE; break; @@ -873,6 +928,13 @@ main (int argc, char **argv) runner.crashes_per_target[n] = _list_prepend (runner.crashes_per_target[n], test); crashed = TRUE; + } else if (target_error) { + target_status[n] = CAIRO_TEST_ERROR; + runner.num_error_per_target[n]++; + runner.errors_per_target[n] = _list_prepend (runner.errors_per_target[n], + test); + + error = TRUE; } else if (target_failed) { target_status[n] = CAIRO_TEST_FAILURE; runner.num_failed_per_target[n]++; @@ -916,6 +978,28 @@ main (int argc, char **argv) } runner.num_crashed++; runner.passed = FALSE; + } else if (error) { + if (! in_preamble) { + len = 0; + for (n = 0 ; n < runner.base.num_targets; n++) { + if (target_status[n] == CAIRO_TEST_ERROR) { + if (strstr (targets, + runner.base.targets_to_test[n]->name) == NULL) + { + len += snprintf (targets + len, + sizeof (targets) - len, + "%s, ", + runner.base.targets_to_test[n]->name); + } + } + } + targets[len-2] = '\0'; + _log (&runner.base, "%s: ERROR (%s)\n", name, targets); + } else { + _log (&runner.base, "%s: ERROR\n", name); + } + runner.num_error++; + runner.passed = FALSE; } else if (failed) { if (! in_preamble) { len = 0; @@ -964,6 +1048,7 @@ main (int argc, char **argv) for (n = 0 ; n < runner.base.num_targets; n++) { runner.crashes_per_target[n] = _list_reverse (runner.crashes_per_target[n]); + runner.errors_per_target[n] = _list_reverse (runner.errors_per_target[n]); runner.fails_per_target[n] = _list_reverse (runner.fails_per_target[n]); } diff --git a/test/cairo-test.c b/test/cairo-test.c index aa8dd3a60..af4487a57 100644 --- a/test/cairo-test.c +++ b/test/cairo-test.c @@ -1444,7 +1444,7 @@ REPEAT: if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) { cairo_test_log (ctx, "Error: Function under test left cairo status in an error state: %s\n", cairo_status_to_string (cairo_status (cr))); - ret = CAIRO_TEST_FAILURE; + ret = CAIRO_TEST_ERROR; goto UNWIND_CAIRO; } @@ -1665,6 +1665,21 @@ _cairo_test_context_run_for_target (cairo_test_context_t *ctx, fail_face, normal_face); break; + case CAIRO_TEST_ERROR: + if (print_fail_on_stdout && ctx->thread == 0) { + printf ("!!!ERROR!!!\n"); + } else { + /* eat the test name */ + printf ("\r"); + fflush (stdout); + } + cairo_test_log (ctx, "ERROR\n"); + fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!ERROR!!!%s\n", + ctx->test_name, target->name, + cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "", + fail_face, normal_face); + break; + case CAIRO_TEST_XFAILURE: if (print_fail_on_stdout && ctx->thread == 0) { printf ("XFAIL\n"); @@ -1732,6 +1747,9 @@ _cairo_test_context_run_for_target (cairo_test_context_t *ctx, case CAIRO_TEST_CRASHED: printf ("!!!CRASHED!!!\n"); break; + case CAIRO_TEST_ERROR: + printf ("!!!ERRORED!!!\n"); + break; case CAIRO_TEST_XFAILURE: printf ("XFAIL\n"); break; diff --git a/test/cairo-test.h b/test/cairo-test.h index 53db80b69..e2bf1a759 100644 --- a/test/cairo-test.h +++ b/test/cairo-test.h @@ -124,6 +124,7 @@ typedef enum cairo_test_status { CAIRO_TEST_FAILURE, CAIRO_TEST_NEW, CAIRO_TEST_XFAILURE, + CAIRO_TEST_ERROR, CAIRO_TEST_CRASHED, CAIRO_TEST_UNTESTED = 77 /* match automake's skipped exit status */ } cairo_test_status_t;