diff --git a/src/evdev.c b/src/evdev.c
index b6913331..1b7d5a83 100644
--- a/src/evdev.c
+++ b/src/evdev.c
@@ -231,8 +231,24 @@ evdev_button_scroll_button(struct evdev_device *device,
 	device->scroll.button_scroll_btn_pressed = is_press;
 
 	if (is_press) {
-		libinput_timer_set(&device->scroll.timer,
-				   time + DEFAULT_MIDDLE_BUTTON_SCROLL_TIMEOUT);
+		enum timer_flags flags = TIMER_FLAG_NONE;
+
+		/* Special case: if middle button emulation is enabled and
+		 * our scroll button is the left or right button, we only
+		 * get here *after* the middle button timeout has expired
+		 * for that button press. The time passed is the button-down
+		 * time though (which is in the past), so we have to allow
+		 * for a negative timer to be set.
+		 */
+		if (device->middlebutton.enabled &&
+		    (device->scroll.button == BTN_LEFT ||
+		     device->scroll.button == BTN_RIGHT)) {
+			flags = TIMER_FLAG_ALLOW_NEGATIVE;
+		}
+
+		libinput_timer_set_flags(&device->scroll.timer,
+					 time + DEFAULT_MIDDLE_BUTTON_SCROLL_TIMEOUT,
+					 flags);
 		device->scroll.button_down_time = time;
 	} else {
 		libinput_timer_cancel(&device->scroll.timer);
diff --git a/src/timer.c b/src/timer.c
index c9247942..362d81db 100644
--- a/src/timer.c
+++ b/src/timer.c
@@ -67,19 +67,23 @@ libinput_timer_arm_timer_fd(struct libinput *libinput)
 }
 
 void
-libinput_timer_set(struct libinput_timer *timer, uint64_t expire)
+libinput_timer_set_flags(struct libinput_timer *timer,
+			 uint64_t expire,
+			 uint32_t flags)
 {
 #ifndef NDEBUG
 	uint64_t now = libinput_now(timer->libinput);
-	if (expire < now)
-		log_bug_libinput(timer->libinput,
-				 "timer offset negative (-%" PRIu64 ")\n",
-				 now - expire);
-	else if ((expire - now) > ms2us(5000))
+	if (expire < now) {
+		if ((flags & TIMER_FLAG_ALLOW_NEGATIVE) == 0)
+			log_bug_libinput(timer->libinput,
+					 "timer offset negative (-%" PRIu64 ")\n",
+					 now - expire);
+	} else if ((expire - now) > ms2us(5000)) {
 		log_bug_libinput(timer->libinput,
 				 "timer offset more than 5s, now %"
 				 PRIu64 " expire %" PRIu64 "\n",
 				 now, expire);
+	}
 #endif
 
 	assert(expire);
@@ -91,6 +95,12 @@ libinput_timer_set(struct libinput_timer *timer, uint64_t expire)
 	libinput_timer_arm_timer_fd(timer->libinput);
 }
 
+void
+libinput_timer_set(struct libinput_timer *timer, uint64_t expire)
+{
+	libinput_timer_set_flags(timer, expire, TIMER_FLAG_NONE);
+}
+
 void
 libinput_timer_cancel(struct libinput_timer *timer)
 {
diff --git a/src/timer.h b/src/timer.h
index f8315cfe..93d684ab 100644
--- a/src/timer.h
+++ b/src/timer.h
@@ -47,6 +47,16 @@ libinput_timer_init(struct libinput_timer *timer, struct libinput *libinput,
 void
 libinput_timer_set(struct libinput_timer *timer, uint64_t expire);
 
+enum timer_flags {
+	TIMER_FLAG_NONE = 0,
+	TIMER_FLAG_ALLOW_NEGATIVE = (1 << 0),
+};
+
+void
+libinput_timer_set_flags(struct libinput_timer *timer,
+			 uint64_t expire,
+			 uint32_t flags);
+
 void
 libinput_timer_cancel(struct libinput_timer *timer);