diff --git a/src/evdev-touchpad.c b/src/evdev-touchpad.c
index 8185bf27..65c5ea05 100644
--- a/src/evdev-touchpad.c
+++ b/src/evdev-touchpad.c
@@ -714,6 +714,7 @@ touchpad_destroy(struct evdev_dispatch *dispatch)
 	struct libinput *libinput = touchpad->device->base.seat->libinput;
 
 	touchpad->filter->interface->destroy(touchpad->filter);
+	close(touchpad->fsm.timer.fd);
 	libinput_remove_source(libinput, touchpad->fsm.timer.source);
 	free(touchpad->fsm.events);
 	free(dispatch);
diff --git a/src/libinput.c b/src/libinput.c
index 58b87b0f..182c401e 100644
--- a/src/libinput.c
+++ b/src/libinput.c
@@ -442,7 +442,6 @@ libinput_remove_source(struct libinput *libinput,
 		       struct libinput_source *source)
 {
 	epoll_ctl(libinput->epoll_fd, EPOLL_CTL_DEL, source->fd, NULL);
-	close(source->fd);
 	source->fd = -1;
 	list_insert(&libinput->source_destroy_list, &source->link);
 }