diff --git a/src/evdev.c b/src/evdev.c
index 642f441c..ed1a9a30 100644
--- a/src/evdev.c
+++ b/src/evdev.c
@@ -442,6 +442,7 @@ evdev_button_scroll_button(struct evdev_device *device,
 	if (is_press) {
 		libinput_timer_set(&device->scroll.timer,
 				time + DEFAULT_MIDDLE_BUTTON_SCROLL_TIMEOUT);
+		device->scroll.button_down_time = time;
 	} else {
 		libinput_timer_cancel(&device->scroll.timer);
 		if (device->scroll.button_scroll_active) {
@@ -451,7 +452,8 @@ evdev_button_scroll_button(struct evdev_device *device,
 		} else {
 			/* If the button is released quickly enough emit the
 			 * button press/release events. */
-			evdev_pointer_notify_physical_button(device, time,
+			evdev_pointer_notify_physical_button(device,
+					device->scroll.button_down_time,
 					device->scroll.button,
 					LIBINPUT_BUTTON_STATE_PRESSED);
 			evdev_pointer_notify_physical_button(device, time,
diff --git a/src/evdev.h b/src/evdev.h
index 69279527..3f63c57a 100644
--- a/src/evdev.h
+++ b/src/evdev.h
@@ -150,6 +150,8 @@ struct evdev_device {
 		/* Currently enabled method, button */
 		enum libinput_config_scroll_method method;
 		uint32_t button;
+		uint64_t button_down_time;
+
 		/* set during device init, used at runtime to delay changes
 		 * until all buttons are up */
 		enum libinput_config_scroll_method want_method;
diff --git a/test/trackpoint.c b/test/trackpoint.c
index 9fcce6f7..0a6f6b0f 100644
--- a/test/trackpoint.c
+++ b/test/trackpoint.c
@@ -35,15 +35,34 @@ START_TEST(trackpoint_middlebutton)
 {
 	struct litest_device *dev = litest_current_device();
 	struct libinput *li = dev->libinput;
+	struct libinput_event *event;
+	struct libinput_event_pointer *ptrev;
+	uint64_t ptime, rtime;
 
 	litest_drain_events(li);
 
 	/* A quick middle button click should get reported normally */
 	litest_button_click(dev, BTN_MIDDLE, 1);
+	msleep(2);
 	litest_button_click(dev, BTN_MIDDLE, 0);
 
-	litest_assert_button_event(li, BTN_MIDDLE, 1);
-	litest_assert_button_event(li, BTN_MIDDLE, 0);
+	litest_wait_for_event(li);
+
+	event = libinput_get_event(li);
+	ptrev = litest_is_button_event(event,
+				       BTN_MIDDLE,
+				       LIBINPUT_BUTTON_STATE_PRESSED);
+	ptime = libinput_event_pointer_get_time(ptrev);
+	libinput_event_destroy(event);
+
+	event = libinput_get_event(li);
+	ptrev = litest_is_button_event(event,
+				       BTN_MIDDLE,
+				       LIBINPUT_BUTTON_STATE_RELEASED);
+	rtime = libinput_event_pointer_get_time(ptrev);
+	libinput_event_destroy(event);
+
+	ck_assert_int_lt(ptime, rtime);
 
 	litest_assert_empty_queue(li);
 }