diff --git a/src/libinput.c b/src/libinput.c
index 84e329de..514a6119 100644
--- a/src/libinput.c
+++ b/src/libinput.c
@@ -3954,13 +3954,13 @@ LIBINPUT_EXPORT enum libinput_config_status
 libinput_device_config_scroll_set_button(struct libinput_device *device,
 					 uint32_t button)
 {
-	if (button && !libinput_device_pointer_has_button(device, button))
-		return LIBINPUT_CONFIG_STATUS_INVALID;
-
 	if ((libinput_device_config_scroll_get_methods(device) &
 	     LIBINPUT_CONFIG_SCROLL_ON_BUTTON_DOWN) == 0)
 		return LIBINPUT_CONFIG_STATUS_UNSUPPORTED;
 
+	if (button && !libinput_device_pointer_has_button(device, button))
+		return LIBINPUT_CONFIG_STATUS_INVALID;
+
 	return device->config.scroll_method->set_button(device, button);
 }
 
diff --git a/test/test-pointer.c b/test/test-pointer.c
index f28b5076..57e7fd14 100644
--- a/test/test-pointer.c
+++ b/test/test-pointer.c
@@ -1014,6 +1014,28 @@ START_TEST(pointer_scroll_button)
 }
 END_TEST
 
+START_TEST(pointer_scroll_button_noscroll)
+{
+	struct litest_device *dev = litest_current_device();
+	struct libinput_device *device = dev->libinput_device;
+	uint32_t methods, button;
+	enum libinput_config_status status;
+
+	methods = libinput_device_config_scroll_get_method(device);
+	ck_assert_int_eq((methods & LIBINPUT_CONFIG_SCROLL_ON_BUTTON_DOWN), 0);
+	button = libinput_device_config_scroll_get_button(device);
+	ck_assert_int_eq(button, 0);
+	button = libinput_device_config_scroll_get_default_button(device);
+	ck_assert_int_eq(button, 0);
+
+	status = libinput_device_config_scroll_set_method(device,
+					LIBINPUT_CONFIG_SCROLL_ON_BUTTON_DOWN);
+	ck_assert_int_eq(status, LIBINPUT_CONFIG_STATUS_UNSUPPORTED);
+	status = libinput_device_config_scroll_set_button(device, BTN_LEFT);
+	ck_assert_int_eq(status, LIBINPUT_CONFIG_STATUS_UNSUPPORTED);
+}
+END_TEST
+
 START_TEST(pointer_scroll_button_no_event_before_timeout)
 {
 	struct litest_device *device = litest_current_device();
@@ -1889,6 +1911,7 @@ litest_setup_tests_pointer(void)
 	litest_add_for_device("pointer:button", pointer_button_has_no_button, LITEST_KEYBOARD);
 	litest_add("pointer:scroll", pointer_scroll_wheel, LITEST_WHEEL, LITEST_TABLET);
 	litest_add("pointer:scroll", pointer_scroll_button, LITEST_RELATIVE|LITEST_BUTTON, LITEST_ANY);
+	litest_add("pointer:scroll", pointer_scroll_button_noscroll, LITEST_ANY, LITEST_RELATIVE|LITEST_BUTTON);
 	litest_add("pointer:scroll", pointer_scroll_button_no_event_before_timeout, LITEST_RELATIVE|LITEST_BUTTON, LITEST_ANY);
 	litest_add("pointer:scroll", pointer_scroll_button_middle_emulation, LITEST_RELATIVE|LITEST_BUTTON, LITEST_ANY);
 	litest_add("pointer:scroll", pointer_scroll_nowheel_defaults, LITEST_RELATIVE|LITEST_BUTTON, LITEST_WHEEL);