Merge branch 'litest-filter-tests'

The litest-selftest has its own main method and compiles litest.c with special
flags. Use that to ifdef out the litest.c main function, and inline the
litest_run/litest_parse_args functions so gcc doesn't complain about unused
functions.

Signed-off-by: Peter Hutterer <peter.hutterer@who-t.net>

doc/test-suite.dox

@@ -25,10 +25,10 @@ resulting `/dev/input/eventX` nodes. Some tests require temporary udev rules.
 
 @section test-filtering Selective running of tests
 
-Check enables tests to be grouped into suites and test cases, litest uses
-test suites as a general feature-specific grouping (e.g. "touchpad:tap") and
-instantiates one test case per device. The --list flag shows the list of
-suites and tests.
+litest's tests are grouped by test groups and devices. A test group is e.g.
+"touchpad:tap" and incorporates all tapping-related tests for touchpads.
+Each test function is (usually) run with one or more specific devices.
+The `--list` commandline argument shows the list of suites and tests.
 @code
 $ ./test/test-device --list
 device:wheel:
@@ -52,18 +52,31 @@ uinput by litest. The "no device" entry signals that litest does not
 instantiate a uinput device for a specific test (though the test itself may
 instantiate one).
 
-Check provides two filters through environment variables: <b>CK_RUN_SUITE</b>
-and <b>CK_RUN_CASE</b>. They may be used independently or combined to narrow
-down the set of tests to run. For example:
+The `--filter-test` argument enables selective running of tests through
+basic shell-style function name matching. For example:
 
 @code
-$ CK_RUN_SUITE="device:wheel" ./test/test-device
-$ CK_RUN_CASE="wheel only" ./test/test-device
-$ CK_RUN_SUITE="device:wheel" CK_RUN_CASE="wheel only" ./test/test-device
+$ ./test/test-touchpad --filter-test="*1fg_tap*"
 @endcode
 
-Check and litest currently do not provide a way to run a specific test
-function only.
+The `--filter-device` argument enables selective running of tests through
+basic shell-style device name matching. The device names matched are the
+litest-specific shortnames, see the output of `--list`. For example:
+
+@code
+$ ./test/test-touchpad --filter-device="synaptics*"
+@endcode
+
+The `--filter-group` argument enables selective running of test groups
+through basic shell-style test group matching. The test groups matched are
+litest-specific test groups, see the output of `--list`. For example:
+
+@code
+$ ./test/test-touchpad --filter-group="touchpad:*hover*"
+@endcode
+
+The `--filter-device` and `--filter-group` arguments can be combined with
+`--list` to show which groups and devices will be affected.
 
 @section test-verbosity Controlling test output
 

test/Makefile.am

@@ -110,7 +110,7 @@ test_device_LDADD = $(TEST_LIBS)
 test_device_LDFLAGS = -no-install
 
 test_litest_selftest_SOURCES = litest-selftest.c litest.c litest-int.h litest.h
-test_litest_selftest_CFLAGS = -DLITEST_DISABLE_BACKTRACE_LOGGING
+test_litest_selftest_CFLAGS = -DLITEST_DISABLE_BACKTRACE_LOGGING -DLITEST_NO_MAIN
 test_litest_selftest_LDADD = $(TEST_LIBS)
 test_litest_selftest_LDFLAGS = -no-install
 

test/device.c

@@ -946,7 +946,8 @@ START_TEST(device_wheel_only)
 }
 END_TEST
 
-int main (int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	struct range abs_range = { 0, ABS_MISC };
 	struct range abs_mt_range = { ABS_MT_SLOT + 1, ABS_CNT };
@@ -987,6 +988,4 @@ int main (int argc, char **argv)
 	litest_add_no_device("device:invalid devices", abs_mt_device_missing_res);
 
 	litest_add("device:wheel", device_wheel_only, LITEST_WHEEL, LITEST_RELATIVE|LITEST_ABSOLUTE);
-
-	return litest_run(argc, argv);
 }

test/keyboard.c

@@ -310,14 +310,12 @@ START_TEST(keyboard_keys_bad_device)
 }
 END_TEST
 
-int
-main(int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	litest_add_no_device("keyboard:seat key count", keyboard_seat_key_count);
 	litest_add_no_device("keyboard:key counting", keyboard_ignore_no_pressed_release);
 	litest_add_no_device("keyboard:key counting", keyboard_key_auto_release);
 	litest_add("keyboard:keys", keyboard_has_key, LITEST_KEYS, LITEST_ANY);
 	litest_add("keyboard:keys", keyboard_keys_bad_device, LITEST_ANY, LITEST_ANY);
-
-	return litest_run(argc, argv);
 }

test/litest.c

@@ -30,6 +30,7 @@
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <fnmatch.h>
 #include <getopt.h>
 #include <poll.h>
 #include <stdint.h>
@@ -51,6 +52,9 @@
 
 static int in_debugger = -1;
 static int verbose = 0;
+const char *filter_test = NULL;
+const char *filter_device = NULL;
+const char *filter_group = NULL;
 
 #ifdef HAVE_LIBUNWIND
 #define UNW_LOCAL_ONLY
@@ -416,6 +420,7 @@ litest_drop_udev_rules(void)
 
 static void
 litest_add_tcase_for_device(struct suite *suite,
+			    const char *funcname,
 			    void *func,
 			    const struct litest_test_device *dev,
 			    const struct range *range)
@@ -463,6 +468,10 @@ litest_add_tcase_no_device(struct suite *suite,
 	struct test *t;
 	const char *test_name = "no device";
 
+	if (filter_device &&
+	    fnmatch(filter_device, test_name, 0) != 0)
+		return;
+
 	list_for_each(t, &suite->tests, node) {
 		if (strcmp(t->name, test_name) != 0)
 			continue;
@@ -483,53 +492,6 @@ litest_add_tcase_no_device(struct suite *suite,
 	suite_add_tcase(suite->suite, t->tc);
 }
 
-static void
-litest_add_tcase(struct suite *suite, void *func,
-		 enum litest_device_feature required,
-		 enum litest_device_feature excluded,
-		 const struct range *range)
-{
-	struct litest_test_device **dev = devices;
-
-	assert(required >= LITEST_DISABLE_DEVICE);
-	assert(excluded >= LITEST_DISABLE_DEVICE);
-
-	if (required == LITEST_DISABLE_DEVICE &&
-	    excluded == LITEST_DISABLE_DEVICE) {
-		litest_add_tcase_no_device(suite, func, range);
-	} else if (required != LITEST_ANY || excluded != LITEST_ANY) {
-		while (*dev) {
-			if (((*dev)->features & required) == required &&
-			    ((*dev)->features & excluded) == 0)
-				litest_add_tcase_for_device(suite, func, *dev, range);
-			dev++;
-		}
-	} else {
-		while (*dev) {
-			litest_add_tcase_for_device(suite, func, *dev, range);
-			dev++;
-		}
-	}
-}
-
-void
-litest_add_no_device(const char *name, void *func)
-{
-	litest_add(name, func, LITEST_DISABLE_DEVICE, LITEST_DISABLE_DEVICE);
-}
-
-void
-litest_add_ranged_no_device(const char *name,
-			    void *func,
-			    const struct range *range)
-{
-	litest_add_ranged(name,
-			  func,
-			  LITEST_DISABLE_DEVICE,
-			  LITEST_DISABLE_DEVICE,
-			  range);
-}
-
 static struct suite *
 get_suite(const char *name)
 {
@@ -554,51 +516,148 @@ get_suite(const char *name)
 	return s;
 }
 
-void
-litest_add(const char *name,
-	   void *func,
-	   enum litest_device_feature required,
-	   enum litest_device_feature excluded)
+static void
+litest_add_tcase(const char *suite_name,
+		 const char *funcname,
+		 void *func,
+		 enum litest_device_feature required,
+		 enum litest_device_feature excluded,
+		 const struct range *range)
 {
-	litest_add_ranged(name, func, required, excluded, NULL);
+	struct litest_test_device **dev = devices;
+	struct suite *suite;
+
+	assert(required >= LITEST_DISABLE_DEVICE);
+	assert(excluded >= LITEST_DISABLE_DEVICE);
+
+	if (filter_test &&
+	    fnmatch(filter_test, funcname, 0) != 0)
+		return;
+
+	if (filter_group &&
+	    fnmatch(filter_group, suite_name, 0) != 0)
+		return;
+
+	suite = get_suite(suite_name);
+
+	if (required == LITEST_DISABLE_DEVICE &&
+	    excluded == LITEST_DISABLE_DEVICE) {
+		litest_add_tcase_no_device(suite, func, range);
+	} else if (required != LITEST_ANY || excluded != LITEST_ANY) {
+		for (; *dev; dev++) {
+			if (filter_device &&
+			    fnmatch(filter_device, (*dev)->shortname, 0) != 0)
+				continue;
+			if (((*dev)->features & required) != required ||
+			    ((*dev)->features & excluded) != 0)
+				continue;
+
+			litest_add_tcase_for_device(suite,
+						    funcname,
+						    func,
+						    *dev,
+						    range);
+		}
+	} else {
+		for (; *dev; dev++) {
+			if (filter_device &&
+			    fnmatch(filter_device, (*dev)->shortname, 0) != 0)
+				continue;
+
+			litest_add_tcase_for_device(suite,
+						    funcname,
+						    func,
+						    *dev,
+						    range);
+		}
+	}
 }
 
 void
-litest_add_ranged(const char *name,
-		  void *func,
-		  enum litest_device_feature required,
-		  enum litest_device_feature excluded,
-		  const struct range *range)
+_litest_add_no_device(const char *name, const char *funcname, void *func)
 {
-	litest_add_tcase(get_suite(name), func, required, excluded, range);
+	_litest_add(name, funcname, func, LITEST_DISABLE_DEVICE, LITEST_DISABLE_DEVICE);
 }
 
 void
-litest_add_for_device(const char *name,
-		      void *func,
-		      enum litest_device_type type)
-{
-	litest_add_ranged_for_device(name, func, type, NULL);
-}
-
-void
-litest_add_ranged_for_device(const char *name,
+_litest_add_ranged_no_device(const char *name,
+			     const char *funcname,
 			     void *func,
-			     enum litest_device_type type,
 			     const struct range *range)
+{
+	_litest_add_ranged(name,
+			   funcname,
+			   func,
+			   LITEST_DISABLE_DEVICE,
+			   LITEST_DISABLE_DEVICE,
+			   range);
+}
+
+void
+_litest_add(const char *name,
+	    const char *funcname,
+	    void *func,
+	    enum litest_device_feature required,
+	    enum litest_device_feature excluded)
+{
+	_litest_add_ranged(name,
+			   funcname,
+			   func,
+			   required,
+			   excluded,
+			   NULL);
+}
+
+void
+_litest_add_ranged(const char *name,
+		   const char *funcname,
+		   void *func,
+		   enum litest_device_feature required,
+		   enum litest_device_feature excluded,
+		   const struct range *range)
+{
+	litest_add_tcase(name, funcname, func, required, excluded, range);
+}
+
+void
+_litest_add_for_device(const char *name,
+		       const char *funcname,
+		       void *func,
+		       enum litest_device_type type)
+{
+	_litest_add_ranged_for_device(name, funcname, func, type, NULL);
+}
+
+void
+_litest_add_ranged_for_device(const char *name,
+			      const char *funcname,
+			      void *func,
+			      enum litest_device_type type,
+			      const struct range *range)
 {
 	struct suite *s;
 	struct litest_test_device **dev = devices;
 
 	assert(type < LITEST_NO_DEVICE);
 
+	if (filter_group &&
+	    fnmatch(filter_group, name, 0) != 0)
+		return;
+
 	s = get_suite(name);
-	while (*dev) {
+	for (; *dev; dev++) {
+		if (filter_device &&
+		    fnmatch(filter_device, (*dev)->shortname, 0) != 0)
+			continue;
+
 		if ((*dev)->type == type) {
-			litest_add_tcase_for_device(s, func, *dev, range);
+			litest_add_tcase_for_device(s,
+						    funcname,
+						    func,
+						    *dev,
+						    range);
 			return;
 		}
-		dev++;
 	}
 
 	litest_abort_msg("Invalid test device type");
@@ -685,13 +744,7 @@ struct libinput_interface interface = {
 	.close_restricted = close_restricted,
 };
 
-static const struct option opts[] = {
-	{ "list", 0, 0, 'l' },
-	{ "verbose", 0, 0, 'v' },
-	{ 0, 0, 0, 0}
-};
-
-int
+static inline int
 litest_run(int argc, char **argv)
 {
 	struct suite *s, *snext;
@@ -711,27 +764,6 @@ litest_run(int argc, char **argv)
 			srunner_add_suite(sr, s->suite);
 	}
 
-	while(1) {
-		int c;
-		int option_index = 0;
-
-		c = getopt_long(argc, argv, "", opts, &option_index);
-		if (c == -1)
-			break;
-		switch(c) {
-			case 'l':
-				litest_list_tests(&all_tests);
-				return 0;
-			case 'v':
-				verbose = 1;
-				break;
-			default:
-				fprintf(stderr, "usage: %s [--list]\n", argv[0]);
-				return 1;
-
-		}
-	}
-
 	if (getenv("LITEST_VERBOSE"))
 		verbose = 1;
 
@@ -2112,3 +2144,67 @@ litest_semi_mt_touch_up(struct litest_device *d,
 
 	litest_event(d, EV_SYN, SYN_REPORT, 0);
 }
+
+static inline int
+litest_parse_argv(int argc, char **argv)
+{
+	enum {
+		OPT_FILTER_TEST,
+		OPT_FILTER_DEVICE,
+		OPT_FILTER_GROUP,
+		OPT_LIST,
+		OPT_VERBOSE,
+	};
+	static const struct option opts[] = {
+		{ "filter-test", 1, 0, OPT_FILTER_TEST },
+		{ "filter-device", 1, 0, OPT_FILTER_DEVICE },
+		{ "filter-group", 1, 0, OPT_FILTER_GROUP },
+		{ "list", 0, 0, OPT_LIST },
+		{ "verbose", 0, 0, OPT_VERBOSE },
+		{ 0, 0, 0, 0}
+	};
+
+	while(1) {
+		int c;
+		int option_index = 0;
+
+		c = getopt_long(argc, argv, "", opts, &option_index);
+		if (c == -1)
+			break;
+		switch(c) {
+		case OPT_FILTER_TEST:
+			filter_test = optarg;
+			break;
+		case OPT_FILTER_DEVICE:
+			filter_device = optarg;
+			break;
+		case OPT_FILTER_GROUP:
+			filter_group = optarg;
+			break;
+		case OPT_LIST:
+			litest_list_tests(&all_tests);
+			exit(0);
+		case OPT_VERBOSE:
+			verbose = 1;
+			break;
+		default:
+			fprintf(stderr, "usage: %s [--list]\n", argv[0]);
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+#ifndef LITEST_NO_MAIN
+int
+main(int argc, char **argv)
+{
+	if (litest_parse_argv(argc, argv) != 0)
+		return EXIT_FAILURE;
+
+	litest_setup_tests();
+
+	return litest_run(argc, argv);
+}
+#endif

test/litest.h

@@ -208,28 +208,47 @@ litest_fail_comparison_ptr(const char *file,
 			   const char *func,
 			   const char *comparison);
 
-void litest_add(const char *name, void *func,
-		enum litest_device_feature required_feature,
-		enum litest_device_feature excluded_feature);
-void litest_add_ranged(const char *name,
-		       void *func,
-		       enum litest_device_feature required,
-		       enum litest_device_feature excluded,
-		       const struct range *range);
-void
-litest_add_for_device(const char *name,
-		      void *func,
-		      enum litest_device_type type);
-void litest_add_ranged_for_device(const char *name,
+#define litest_add(name_, func_, ...) \
+	_litest_add(name_, #func_, func_, __VA_ARGS__)
+#define litest_add_ranged(name_, func_, ...) \
+	_litest_add_ranged(name_, #func_, func_, __VA_ARGS__)
+#define litest_add_for_device(name_, func_, ...) \
+	_litest_add_for_device(name_, #func_, func_, __VA_ARGS__)
+#define litest_add_ranged_for_device(name_, func_, ...) \
+	_litest_add_ranged_for_device(name_, #func_, func_, __VA_ARGS__)
+#define litest_add_no_device(name_, func_) \
+	_litest_add_no_device(name_, #func_, func_)
+#define litest_add_ranged_no_device(name_, func_, ...) \
+	_litest_add_ranged_no_device(name_, #func_, func_, __VA_ARGS__)
+void _litest_add(const char *name,
+		 const char *funcname,
+		 void *func,
+		 enum litest_device_feature required_feature,
+		 enum litest_device_feature excluded_feature);
+void _litest_add_ranged(const char *name,
+			const char *funcname,
+			void *func,
+			enum litest_device_feature required,
+			enum litest_device_feature excluded,
+			const struct range *range);
+void _litest_add_for_device(const char *name,
+			    const char *funcname,
+			    void *func,
+			    enum litest_device_type type);
+void _litest_add_ranged_for_device(const char *name,
+				   const char *funcname,
+				   void *func,
+				   enum litest_device_type type,
+				   const struct range *range);
+void _litest_add_no_device(const char *name,
+			   const char *funcname,
+			   void *func);
+void _litest_add_ranged_no_device(const char *name,
+				  const char *funcname,
 				  void *func,
-				  enum litest_device_type type,
 				  const struct range *range);
-void litest_add_no_device(const char *name, void *func);
-void litest_add_ranged_no_device(const char *name,
-				 void *func,
-				 const struct range *range);
 
-int litest_run(int argc, char **argv);
+extern void litest_setup_tests(void);
 struct litest_device * litest_create_device(enum litest_device_type which);
 struct litest_device * litest_add_device(struct libinput *libinput,
 					 enum litest_device_type which);

test/log.c

@@ -139,12 +139,11 @@ START_TEST(log_priority)
 }
 END_TEST
 
-int main (int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	litest_add_no_device("log:defaults", log_default_priority);
 	litest_add_no_device("log:logging", log_handler_invoked);
 	litest_add_no_device("log:logging", log_handler_NULL);
 	litest_add_no_device("log:logging", log_priority);
-
-	return litest_run(argc, argv);
 }

test/misc.c

@@ -583,7 +583,8 @@ START_TEST(trackpoint_accel_parser)
 }
 END_TEST
 
-int main (int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	litest_add_no_device("events:conversion", event_conversion_device_notify);
 	litest_add_for_device("events:conversion", event_conversion_pointer, LITEST_MOUSE);
@@ -600,6 +601,4 @@ int main (int argc, char **argv)
 	litest_add_no_device("misc:dpi parser", dpi_parser);
 	litest_add_no_device("misc:wheel click parser", wheel_click_parser);
 	litest_add_no_device("misc:trackpoint accel parser", trackpoint_accel_parser);
-
-	return litest_run(argc, argv);
 }

test/path.c

@@ -874,8 +874,8 @@ START_TEST(path_seat_recycle)
 }
 END_TEST
 
-int
-main(int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	litest_add_no_device("path:create", path_create_NULL);
 	litest_add_no_device("path:create", path_create_invalid);
@@ -896,6 +896,4 @@ main(int argc, char **argv)
 	litest_add_for_device("path:device events", path_remove_device, LITEST_SYNAPTICS_CLICKPAD);
 	litest_add_for_device("path:device events", path_double_remove_device, LITEST_SYNAPTICS_CLICKPAD);
 	litest_add_no_device("path:seat", path_seat_recycle);
-
-	return litest_run(argc, argv);
 }

test/pointer.c

@@ -1318,7 +1318,8 @@ START_TEST(middlebutton_default_disabled)
 }
 END_TEST
 
-int main (int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	struct range axis_range = {ABS_X, ABS_Y + 1};
 
@@ -1360,6 +1361,4 @@ int main (int argc, char **argv)
 	litest_add("pointer:middlebutton", middlebutton_default_disabled, LITEST_ANY, LITEST_BUTTON);
 
 	litest_add_ranged("pointer:state", pointer_absolute_initial_state, LITEST_ABSOLUTE, LITEST_ANY, &axis_range);
-
-	return litest_run(argc, argv);
 }

test/touch.c

@@ -650,8 +650,8 @@ START_TEST(touch_initial_state)
 }
 END_TEST
 
-int
-main(int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	struct range axes = { ABS_X, ABS_Y + 1};
 
@@ -676,6 +676,4 @@ main(int argc, char **argv)
 	litest_add("touch:protocol a", touch_protocol_a_2fg_touch, LITEST_PROTOCOL_A, LITEST_ANY);
 
 	litest_add_ranged("touch:state", touch_initial_state, LITEST_TOUCH, LITEST_PROTOCOL_A, &axes);
-
-	return litest_run(argc, argv);
 }

test/touchpad.c

@@ -4590,7 +4590,8 @@ START_TEST(touchpad_initial_state)
 }
 END_TEST
 
-int main(int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	struct range multitap_range = {3, 8};
 	struct range axis_range = {ABS_X, ABS_Y + 1};
@@ -4737,6 +4738,4 @@ int main(int argc, char **argv)
 	litest_add_for_device("touchpad:trackpoint", touchpad_trackpoint_no_trackpoint, LITEST_SYNAPTICS_TRACKPOINT_BUTTONS);
 
 	litest_add_ranged("touchpad:state", touchpad_initial_state, LITEST_TOUCHPAD, LITEST_ANY, &axis_range);
-
-	return litest_run(argc, argv);
 }

test/trackpoint.c

@@ -130,12 +130,11 @@ START_TEST(trackpoint_scroll_source)
 }
 END_TEST
 
-int main(int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	litest_add("trackpoint:middlebutton", trackpoint_middlebutton, LITEST_POINTINGSTICK, LITEST_ANY);
 	litest_add("trackpoint:middlebutton", trackpoint_middlebutton_noscroll, LITEST_POINTINGSTICK, LITEST_ANY);
 	litest_add("trackpoint:scroll", trackpoint_scroll, LITEST_POINTINGSTICK, LITEST_ANY);
 	litest_add("trackpoint:scroll", trackpoint_scroll_source, LITEST_POINTINGSTICK, LITEST_ANY);
-
-	return litest_run(argc, argv);
 }

test/udev.c

@@ -502,8 +502,8 @@ START_TEST(udev_seat_recycle)
 }
 END_TEST
 
-int
-main(int argc, char **argv)
+void
+litest_setup_tests(void)
 {
 	litest_add_no_device("udev:create", udev_create_NULL);
 	litest_add_no_device("udev:create", udev_create_seat0);
@@ -518,6 +518,4 @@ main(int argc, char **argv)
 	litest_add_for_device("udev:suspend", udev_suspend_resume, LITEST_SYNAPTICS_CLICKPAD);
 	litest_add_for_device("udev:device events", udev_device_sysname, LITEST_SYNAPTICS_CLICKPAD);
 	litest_add_for_device("udev:seat", udev_seat_recycle, LITEST_SYNAPTICS_CLICKPAD);
-
-	return litest_run(argc, argv);
 }