fdo-mirrors/cairo
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/cairo/cairo.git synced 2026-02-15 06:50:31 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

test: Distinguish tests that throw an error from a normal fail.

Browse source 
Hitting an error in a test case is almost as bad as crashing, and the
severity may be lost amidst "normal" failures. So introduce a new class
of ERROR so that we can immediately spot these during a test run, and
appropriately log them afterwards.
This commit is contained in:
Chris Wilson 2010-06-11 09:12:16 +01:00
parent eeafeebd2e
commit bd3d9ef3d1
3 changed files with 107 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
test/cairo-test-runner.c
View file

@ -78,15 +78,19 @@ typedef struct _cairo_test_runner {
int num_skipped;
int num_failed;
int num_xfailed;
int num_error;
int num_crashed;
cairo_test_list_t *crashes_preamble;
cairo_test_list_t *errors_preamble;
cairo_test_list_t *fails_preamble;
cairo_test_list_t **crashes_per_target;
cairo_test_list_t **errors_per_target;
cairo_test_list_t **fails_per_target;
int *num_failed_per_target;
int *num_error_per_target;
int *num_crashed_per_target;
cairo_bool_t foreground;
@ -384,13 +388,18 @@ _runner_init (cairo_test_runner_t *runner)
runner->fails_preamble = NULL;
runner->crashes_preamble = NULL;
runner->errors_preamble = NULL;
runner->fails_per_target = xcalloc (sizeof (cairo_test_list_t *),
runner->base.num_targets);
runner->crashes_per_target = xcalloc (sizeof (cairo_test_list_t *),
runner->base.num_targets);
runner->errors_per_target = xcalloc (sizeof (cairo_test_list_t *),
runner->base.num_targets);
runner->num_failed_per_target = xcalloc (sizeof (int),
runner->base.num_targets);
runner->num_error_per_target = xcalloc (sizeof (int),
runner->base.num_targets);
runner->num_crashed_per_target = xcalloc (sizeof (int),
runner->base.num_targets);
}
@ -443,6 +452,21 @@ _runner_print_details (cairo_test_runner_t *runner)
}
_log (&runner->base, "\n");
}
if (runner->errors_preamble) {
int count = 0;
for (list = runner->errors_preamble; list != NULL; list = list->next)
count++;
_log (&runner->base, "Preamble: %d error -", count);
for (list = runner->errors_preamble; list != NULL; list = list->next) {
char *name = cairo_test_get_name (list->test);
_log (&runner->base, " %s", name);
free (name);
}
_log (&runner->base, "\n");
}
if (runner->fails_preamble) {
int count = 0;
@ -479,6 +503,22 @@ _runner_print_details (cairo_test_runner_t *runner)
}
_log (&runner->base, "\n");
}
if (runner->num_error_per_target[n]) {
_log (&runner->base, "%s (%s): %d error -",
target->name,
cairo_boilerplate_content_name (target->content),
runner->num_error_per_target[n]);
for (list = runner->errors_per_target[n];
list != NULL;
list = list->next)
{
char *name = cairo_test_get_name (list->test);
_log (&runner->base, " %s", name);
free (name);
}
_log (&runner->base, "\n");
}
if (runner->num_failed_per_target[n]) {
_log (&runner->base, "%s (%s): %d failed -",
@ -519,16 +559,20 @@ _runner_fini (cairo_test_runner_t *runner)
unsigned int n;
_list_free (runner->crashes_preamble);
_list_free (runner->errors_preamble);
_list_free (runner->fails_preamble);
for (n = 0; n < runner->base.num_targets; n++) {
_list_free (runner->crashes_per_target[n]);
_list_free (runner->errors_per_target[n]);
_list_free (runner->fails_per_target[n]);
}
free (runner->crashes_per_target);
free (runner->errors_per_target);
free (runner->fails_per_target);
free (runner->num_crashed_per_target);
free (runner->num_error_per_target);
free (runner->num_failed_per_target);
cairo_test_fini (&runner->base);
@ -700,7 +744,7 @@ main (int argc, char **argv)
for (test = tests; test != NULL; test = test->next) {
cairo_test_context_t ctx;
cairo_test_status_t status;
cairo_bool_t failed = FALSE, xfailed = FALSE, crashed = FALSE, skipped = TRUE;
cairo_bool_t failed = FALSE, xfailed = FALSE, error = FALSE, crashed = FALSE, skipped = TRUE;
cairo_bool_t in_preamble = FALSE;
char *name = cairo_test_get_name (test);
int i;
@ -810,6 +854,13 @@ main (int argc, char **argv)
failed = TRUE;
goto TEST_DONE;
case CAIRO_TEST_ERROR:
runner.errors_preamble = _list_prepend (runner.errors_preamble,
test);
in_preamble = TRUE;
failed = TRUE;
goto TEST_DONE;
case CAIRO_TEST_NO_MEMORY:
case CAIRO_TEST_CRASHED:
runner.crashes_preamble = _list_prepend (runner.crashes_preamble,
@ -830,6 +881,7 @@ main (int argc, char **argv)
const cairo_boilerplate_target_t *target;
cairo_bool_t target_failed = FALSE,
target_xfailed = FALSE,
target_error = FALSE,
target_crashed = FALSE,
target_skipped = TRUE;
int has_similar;
@ -853,11 +905,14 @@ main (int argc, char **argv)
case CAIRO_TEST_XFAILURE:
target_xfailed = TRUE;
break;
case CAIRO_TEST_NO_MEMORY:
case CAIRO_TEST_NEW:
case CAIRO_TEST_FAILURE:
target_failed = TRUE;
break;
case CAIRO_TEST_ERROR:
target_error = TRUE;
break;
case CAIRO_TEST_NO_MEMORY:
case CAIRO_TEST_CRASHED:
target_crashed = TRUE;
break;
@ -873,6 +928,13 @@ main (int argc, char **argv)
runner.crashes_per_target[n] = _list_prepend (runner.crashes_per_target[n],
test);
crashed = TRUE;
} else if (target_error) {
target_status[n] = CAIRO_TEST_ERROR;
runner.num_error_per_target[n]++;
runner.errors_per_target[n] = _list_prepend (runner.errors_per_target[n],
test);
error = TRUE;
} else if (target_failed) {
target_status[n] = CAIRO_TEST_FAILURE;
runner.num_failed_per_target[n]++;
@ -916,6 +978,28 @@ main (int argc, char **argv)
}
runner.num_crashed++;
runner.passed = FALSE;
} else if (error) {
if (! in_preamble) {
len = 0;
for (n = 0 ; n < runner.base.num_targets; n++) {
if (target_status[n] == CAIRO_TEST_ERROR) {
if (strstr (targets,
runner.base.targets_to_test[n]->name) == NULL)
{
len += snprintf (targets + len,
sizeof (targets) - len,
"%s, ",
runner.base.targets_to_test[n]->name);
}
}
}
targets[len-2] = '\0';
_log (&runner.base, "%s: ERROR (%s)\n", name, targets);
} else {
_log (&runner.base, "%s: ERROR\n", name);
}
runner.num_error++;
runner.passed = FALSE;
} else if (failed) {
if (! in_preamble) {
len = 0;
@ -964,6 +1048,7 @@ main (int argc, char **argv)
for (n = 0 ; n < runner.base.num_targets; n++) {
runner.crashes_per_target[n] = _list_reverse (runner.crashes_per_target[n]);
runner.errors_per_target[n] = _list_reverse (runner.errors_per_target[n]);
runner.fails_per_target[n] = _list_reverse (runner.fails_per_target[n]);
}

20
test/cairo-test.c
View file

@ -1444,7 +1444,7 @@ REPEAT:
if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) {
cairo_test_log (ctx, "Error: Function under test left cairo status in an error state: %s\n",
cairo_status_to_string (cairo_status (cr)));
ret = CAIRO_TEST_FAILURE;
ret = CAIRO_TEST_ERROR;
goto UNWIND_CAIRO;
}
@ -1665,6 +1665,21 @@ _cairo_test_context_run_for_target (cairo_test_context_t *ctx,
fail_face, normal_face);
break;
case CAIRO_TEST_ERROR:
if (print_fail_on_stdout && ctx->thread == 0) {
printf ("!!!ERROR!!!\n");
} else {
/* eat the test name */
printf ("\r");
fflush (stdout);
}
cairo_test_log (ctx, "ERROR\n");
fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!ERROR!!!%s\n",
ctx->test_name, target->name,
cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
fail_face, normal_face);
break;
case CAIRO_TEST_XFAILURE:
if (print_fail_on_stdout && ctx->thread == 0) {
printf ("XFAIL\n");
@ -1732,6 +1747,9 @@ _cairo_test_context_run_for_target (cairo_test_context_t *ctx,
case CAIRO_TEST_CRASHED:
printf ("!!!CRASHED!!!\n");
break;
case CAIRO_TEST_ERROR:
printf ("!!!ERRORED!!!\n");
break;
case CAIRO_TEST_XFAILURE:
printf ("XFAIL\n");
break;

1
test/cairo-test.h
View file

@ -124,6 +124,7 @@ typedef enum cairo_test_status {
CAIRO_TEST_FAILURE,
CAIRO_TEST_NEW,
CAIRO_TEST_XFAILURE,
CAIRO_TEST_ERROR,
CAIRO_TEST_CRASHED,
CAIRO_TEST_UNTESTED = 77 /* match automake's skipped exit status */
} cairo_test_status_t;