diff --git a/hw/xwayland/xwayland-input.c b/hw/xwayland/xwayland-input.c
index cd2046e1d..88977f647 100644
--- a/hw/xwayland/xwayland-input.c
+++ b/hw/xwayland/xwayland-input.c
@@ -527,6 +527,7 @@ pointer_handle_enter(void *data, struct wl_pointer *pointer,
     int sx, sy;
     int dx, dy;
     ScreenPtr pScreen = xwl_screen->screen;
+    xwl_seat->pointer_enter_count++;
     ValuatorMask mask;
 
     /* There's a race here where if we create and then immediately
@@ -618,6 +619,9 @@ pointer_handle_leave(void *data, struct wl_pointer *pointer,
     Bool focus_lost = FALSE;
 
     xwl_screen->serial = serial;
+    BUG_WARN(xwl_seat->pointer_enter_count == 0);
+    if (xwl_seat->pointer_enter_count > 0)
+        xwl_seat->pointer_enter_count--;
 
     /* The pointer has left a known xwindow, save it for a possible match
      * in sprite_check_lost_focus()
@@ -3223,6 +3227,7 @@ sprite_check_lost_focus(SpritePtr sprite, WindowPtr window)
 {
     DeviceIntPtr device, master;
     struct xwl_seat *xwl_seat;
+    Bool pointer_crossing;
 
     for (device = inputInfo.devices; device; device = device->next) {
         /* Ignore non-wayland devices */
@@ -3238,16 +3243,17 @@ sprite_check_lost_focus(SpritePtr sprite, WindowPtr window)
     if (!xwl_seat)
         return FALSE;
 
+    pointer_crossing = (xwl_seat->pointer_enter_count > 0);
     master = GetMaster(device, POINTER_OR_FLOAT);
     if (!master || !master->lastSlave)
-        return FALSE;
+        return !pointer_crossing;
 
     /* We do want the last active slave, we only check on slave xwayland
      * devices so we can find out the xwl_seat, but those don't actually own
      * their sprite, so the match doesn't mean a lot.
      */
     if (master->lastSlave != get_pointer_device(xwl_seat))
-        return FALSE;
+        return !pointer_crossing;
 
     /* If we left the surface with a button down, it means the wayland compositor
      * has grabbed the pointer so we will not get button release events from the
@@ -3269,7 +3275,7 @@ sprite_check_lost_focus(SpritePtr sprite, WindowPtr window)
          IsParent(xwl_seat->last_focus_window->toplevel, window)))
         return TRUE;
 
-    return FALSE;
+    return !pointer_crossing;
 }
 
 static WindowPtr
diff --git a/hw/xwayland/xwayland-input.h b/hw/xwayland/xwayland-input.h
index 0a742f017..02319e297 100644
--- a/hw/xwayland/xwayland-input.h
+++ b/hw/xwayland/xwayland-input.h
@@ -76,6 +76,7 @@ struct xwl_seat {
     struct xwl_window *tablet_focus_window;
     uint32_t id;
     uint32_t pointer_enter_serial;
+    uint8_t pointer_enter_count;
     struct xorg_list link;
     CursorPtr x_cursor;
     OsTimerPtr x_cursor_timer;