diff --git a/src/evdev-fallback.c b/src/evdev-fallback.c
index 13e130ac..4da07e7d 100644
--- a/src/evdev-fallback.c
+++ b/src/evdev-fallback.c
@@ -949,6 +949,8 @@ fallback_lid_toggle_keyboard_listener(struct fallback_dispatch *dispatch,
 {
 	assert(kbd->device);
 
+	libinput_device_remove_event_listener(&kbd->listener);
+
 	if (is_closed) {
 		libinput_device_add_event_listener(
 					&kbd->device->base,
@@ -956,10 +958,7 @@ fallback_lid_toggle_keyboard_listener(struct fallback_dispatch *dispatch,
 					fallback_lid_keyboard_event,
 					dispatch);
 	} else {
-		libinput_device_remove_event_listener(
-					&kbd->listener);
-		libinput_device_init_event_listener(
-					&kbd->listener);
+		libinput_device_init_event_listener(&kbd->listener);
 	}
 }
 
@@ -992,11 +991,11 @@ fallback_process_switch(struct fallback_dispatch *dispatch,
 	case SW_LID:
 		is_closed = !!e->value;
 
+		fallback_lid_toggle_keyboard_listeners(dispatch, is_closed);
+
 		if (dispatch->lid.is_closed == is_closed)
 			return;
 
-		fallback_lid_toggle_keyboard_listeners(dispatch, is_closed);
-
 		dispatch->lid.is_closed = is_closed;
 		fallback_lid_notify_toggle(dispatch, device, time);
 		break;
diff --git a/test/test-switch.c b/test/test-switch.c
index 3a499bf7..d62ca9fa 100644
--- a/test/test-switch.c
+++ b/test/test-switch.c
@@ -513,30 +513,32 @@ START_TEST(lid_open_on_key)
 
 	keyboard = litest_add_device(li, LITEST_KEYBOARD);
 
-	litest_switch_action(sw,
-			     LIBINPUT_SWITCH_LID,
-			     LIBINPUT_SWITCH_STATE_ON);
-	litest_drain_events(li);
+	for (int i = 0; i < 3; i++) {
+		litest_switch_action(sw,
+				     LIBINPUT_SWITCH_LID,
+				     LIBINPUT_SWITCH_STATE_ON);
+		litest_drain_events(li);
 
-	litest_event(keyboard, EV_KEY, KEY_A, 1);
-	litest_event(keyboard, EV_SYN, SYN_REPORT, 0);
-	litest_event(keyboard, EV_KEY, KEY_A, 0);
-	litest_event(keyboard, EV_SYN, SYN_REPORT, 0);
-	libinput_dispatch(li);
+		litest_event(keyboard, EV_KEY, KEY_A, 1);
+		litest_event(keyboard, EV_SYN, SYN_REPORT, 0);
+		litest_event(keyboard, EV_KEY, KEY_A, 0);
+		litest_event(keyboard, EV_SYN, SYN_REPORT, 0);
+		libinput_dispatch(li);
 
-	event = libinput_get_event(li);
-	litest_is_switch_event(event,
-			       LIBINPUT_SWITCH_LID,
-			       LIBINPUT_SWITCH_STATE_OFF);
+		event = libinput_get_event(li);
+		litest_is_switch_event(event,
+				       LIBINPUT_SWITCH_LID,
+				       LIBINPUT_SWITCH_STATE_OFF);
+		libinput_event_destroy(event);
 
-	litest_assert_only_typed_events(li, LIBINPUT_EVENT_KEYBOARD_KEY);
+		litest_assert_only_typed_events(li, LIBINPUT_EVENT_KEYBOARD_KEY);
 
-	litest_switch_action(sw,
-			     LIBINPUT_SWITCH_LID,
-			     LIBINPUT_SWITCH_STATE_OFF);
-	litest_assert_empty_queue(li);
+		litest_switch_action(sw,
+				     LIBINPUT_SWITCH_LID,
+				     LIBINPUT_SWITCH_STATE_OFF);
+		litest_assert_empty_queue(li);
+	}
 
-	libinput_event_destroy(event);
 	litest_delete_device(keyboard);
 }
 END_TEST