tests: Don't abort on failure

Setup a custom assertion handler that increases an assertion counter
instead of aborting on failure. This allows to run all the sub tests
defined for a test and to correctly report which sub test failed.

Signed-off-by: Loïc Molinari <loic.molinari@collabora.com>

tests/assert-test.c

@@ -26,27 +26,9 @@
 #include "config.h"
 
 #include <stdlib.h>
-#include <stdarg.h>
 
-#include "shared/weston-assert.h"
 #include "weston-test-runner.h"
-
-__attribute__((format(printf, 2, 3)))
-static inline void
-test_assert_report(const struct weston_compositor *compositor,
-		   const char *fmt, ...)
-{
-	va_list ap;
-
-	va_start(ap, fmt);
-	vfprintf(stderr, fmt, ap);
-	va_end(ap);
-}
-
-#ifdef custom_assert_fail_
-#undef custom_assert_fail_
-#endif
-#define custom_assert_fail_ test_assert_report
+#include "weston-test-assert.h"
 
 static void
 abort_if_not(bool cond)
@@ -154,4 +136,8 @@ TEST(asserts)
 
 	ret = weston_assert_legal_bits(compositor, val, UINT64_MAX);
 	abort_if_not(ret);
+
+	/* If we reach that point, it's a success so reset the assert counter
+	 * that's been incremented to check that assertions work. */
+	weston_assert_counter_reset();
 }

tests/meson.build

@@ -74,7 +74,7 @@ lib_lcms_util = static_library(
 	[ 'lcms_util.c' ],
 	include_directories: common_inc,
 	dependencies: [
-		dep_lcms2, dep_libm, dep_wayland_server
+		dep_lcms2, dep_libm, dep_wayland_server, dep_pixman
 	],
 	build_by_default: false,
 	install: false,

tests/weston-test-assert.h

@@ -28,8 +28,36 @@
 
 #include <errno.h>
 
+#include "libweston/libweston-internal.h"
 #include "shared/weston-assert.h"
 
+int
+weston_assert_counter_get(void);
+
+void
+weston_assert_counter_inc(void);
+
+void
+weston_assert_counter_reset(void);
+
+__attribute__((format(printf, 2, 3)))
+static inline void
+test_assert_fail(void *compositor, const char *fmt, ...)
+{
+	va_list ap;
+
+	va_start(ap, fmt);
+	vfprintf(stderr, fmt, ap);
+	va_end(ap);
+
+	weston_assert_counter_inc();
+}
+
+#ifdef custom_assert_fail_
+#undef custom_assert_fail_
+#endif
+#define custom_assert_fail_ test_assert_fail
+
 /* Boolean asserts. */
 
 #define test_assert_true(a)  weston_assert_(NULL, a, true,  bool, "%d", ==)
@@ -142,6 +170,12 @@
 #define test_assert_bit_not_set(a, bit) weston_assert_bit_is_not_set(NULL, a, bit)
 #define test_assert_errno(a)            test_assert_int_eq(a, errno)
 #define test_assert_enum(a, b)          test_assert_u64_eq(a, b)
-#define test_assert_not_reached(reason) weston_assert_not_reached(NULL, reason)
+
+/* Explicitly abort when reached. */
+#define test_assert_not_reached(reason) \
+do { \
+	weston_assert_not_reached(NULL, reason); \
+	abort(); \
+} while (0)
 
 #endif /* _WESTON_TEST_ASSERT_H_ */

tests/weston-test-runner.c

@@ -59,6 +59,7 @@ struct weston_test_run_info {
 };
 
 static const struct weston_test_run_info *test_run_info_;
+static int assert_counter_ = 0;
 
 /** Get the test name string with counter
  *
@@ -112,6 +113,24 @@ testlog(const char *fmt, ...)
 	va_end(argp);
 }
 
+int
+weston_assert_counter_get(void)
+{
+	return assert_counter_;
+}
+
+void
+weston_assert_counter_inc(void)
+{
+	assert_counter_++;
+}
+
+void
+weston_assert_counter_reset(void)
+{
+	assert_counter_ = 0;
+}
+
 static const void *
 fixture_setup_array_get_arg(const struct fixture_setup_array *fsa, int findex)
 {
@@ -165,16 +184,13 @@ run_test(struct wet_testsuite_data *suite_data, int fixture_nr,
 	}
 	info.fixture_nr = fixture_nr;
 
+	weston_assert_counter_reset();
+
 	test_run_info_ = &info;
 	t->run(suite_data, data);
 	test_run_info_ = NULL;
 
-	/*
-	 * XXX: We should return t->run(data); but that requires changing
-	 * the function signature and stop using assert() in tests.
-	 * https://gitlab.freedesktop.org/wayland/weston/issues/311
-	 */
-	return RESULT_OK;
+	return weston_assert_counter_get() ? RESULT_FAIL: RESULT_OK;
 }
 
 static void

tests/weston-test.c

@@ -229,7 +229,6 @@ test_seat_release(struct weston_test *test)
 
 	assert(test->is_seat_initialized &&
 	       "Trying to release already released test seat");
-
 	test->is_seat_initialized = false;
 	weston_seat_release(&test->seat);
 	memset(&test->seat, 0, sizeof test->seat);

tests/weston-testsuite-data.h

@@ -26,11 +26,10 @@
 #ifndef WESTON_TESTSUITE_DATA_H
 #define WESTON_TESTSUITE_DATA_H
 
+#include <assert.h>
 #include <errno.h>
 #include <semaphore.h>
 
-#include "weston-test-assert.h"
-
 /** Standard return codes
  *
  * Both Autotools and Meson use these codes as test program exit codes