diff --git a/Xi/xipassivegrab.c b/Xi/xipassivegrab.c
index 911691a2e..a75559e8e 100644
--- a/Xi/xipassivegrab.c
+++ b/Xi/xipassivegrab.c
@@ -137,12 +137,6 @@ ProcXIPassiveGrabDevice(ClientPtr client)
         return BadValue;
     }
 
-    /* XI2 allows 32-bit keycodes but thanks to XKB we can never
-     * implement this. Just return an error for all keycodes that
-     * cannot work anyway, same for buttons > 255. */
-    if (stuff->detail > 255)
-        return XIAlreadyGrabbed;
-
     if (XICheckInvalidMaskBits(client, (unsigned char *) &stuff[1],
                                stuff->mask_len * 4) != Success)
         return BadValue;
@@ -202,6 +196,14 @@ ProcXIPassiveGrabDevice(ClientPtr client)
     for (i = 0; i < stuff->num_modifiers; i++, modifiers++) {
         uint8_t status = Success;
 
+        /* XI2 allows 32-bit keycodes but thanks to XKB we can never
+         * implement this. Pretend that all keycodes above 255 are
+         * already grabbed, same for buttons > 255. */
+        if (stuff->detail > 255) {
+            status = XIAlreadyGrabbed;
+            goto modifier_done;
+        }
+
         param.modifiers = *modifiers;
         ret = CheckGrabValues(client, &param);
         if (ret != Success)
@@ -234,6 +236,7 @@ ProcXIPassiveGrabDevice(ClientPtr client)
             break;
         }
 
+modifier_done:
         if (status != GrabSuccess) {
             xXIGrabModifierInfo *info = modifiers_failed + rep.num_modifiers;