diff --git a/src/evdev-mt-touchpad.c b/src/evdev-mt-touchpad.c
index 3080e8a8..4ebadc25 100644
--- a/src/evdev-mt-touchpad.c
+++ b/src/evdev-mt-touchpad.c
@@ -345,22 +345,26 @@ tp_maybe_end_touch(struct tp_dispatch *tp,
 	switch (t->state) {
 	case TOUCH_NONE:
 	case TOUCH_MAYBE_END:
-	case TOUCH_HOVERING:
 		return;
 	case TOUCH_END:
 		evdev_log_bug_libinput(tp->device,
 				       "touch  already in TOUCH_END\n");
 		return;
+	case TOUCH_HOVERING:
 	case TOUCH_BEGIN:
 	case TOUCH_UPDATE:
 		break;
 	}
 
-	t->dirty = true;
-	t->state = TOUCH_MAYBE_END;
+	if (t->state != TOUCH_HOVERING) {
+		assert(tp->nfingers_down >= 1);
+		tp->nfingers_down--;
+		t->state = TOUCH_MAYBE_END;
+	} else {
+		t->state = TOUCH_NONE;
+	}
 
-	assert(tp->nfingers_down >= 1);
-	tp->nfingers_down--;
+	t->dirty = true;
 }
 
 /**