diff --git a/backend/drm/backend.c b/backend/drm/backend.c
index c2806152b..3f2395030 100644
--- a/backend/drm/backend.c
+++ b/backend/drm/backend.c
@@ -65,7 +65,9 @@ static void backend_destroy(struct wlr_backend *backend) {
 	}
 
 	free(drm->name);
-	wlr_session_close_file(drm->session, drm->dev);
+	if (drm->dev) {
+		wlr_session_close_file(drm->session, drm->dev);
+	}
 	wl_event_source_remove(drm->drm_event);
 	free(drm);
 }
@@ -152,6 +154,7 @@ static void handle_dev_remove(struct wl_listener *listener, void *data) {
 	struct wlr_drm_backend *drm = wl_container_of(listener, drm, dev_remove);
 
 	wlr_log(WLR_INFO, "Destroying DRM backend for %s", drm->name);
+	drm->dev = NULL;
 	backend_destroy(&drm->backend);
 }
 
diff --git a/backend/session/session.c b/backend/session/session.c
index a942e35f6..05937dfff 100644
--- a/backend/session/session.c
+++ b/backend/session/session.c
@@ -227,11 +227,11 @@ static int handle_udev_event(int fd, uint32_t mask, void *data) {
 			}
 		}
 	} else if (strcmp(action, "remove") == 0) {
-		struct wlr_device *dev;
-		wl_list_for_each(dev, &session->devices, link) {
+		struct wlr_device *dev, *tmp;
+		wl_list_for_each_safe(dev, tmp, &session->devices, link) {
 			if (dev->dev == devnum) {
 				wlr_log(WLR_DEBUG, "DRM device %s removed", sysname);
-				wl_signal_emit_mutable(&dev->events.remove, NULL);
+				wlr_session_close_file(session, dev);
 				break;
 			}
 		}
@@ -375,11 +375,10 @@ void wlr_session_close_file(struct wlr_session *session,
 		wlr_log_errno(WLR_ERROR, "Failed to close device %d", dev->device_id);
 	}
 
+	wl_signal_emit_mutable(&dev->events.remove, NULL);
+
 	assert(wl_list_empty(&dev->events.change.listener_list));
-	// TODO: assert that the "remove" listener list is empty as well. Listeners
-	// will typically call wlr_session_close_file() in response, and
-	// wl_signal_emit_mutable() installs two phantom listeners, so we'd count
-	// these two.
+	assert(wl_list_empty(&dev->events.remove.listener_list));
 
 	close(dev->fd);
 	wl_list_remove(&dev->link);