diff --git a/src/gallium/drivers/panfrost/pan_job.c b/src/gallium/drivers/panfrost/pan_job.c
index 4074ae2cbd7..898ba2a26cb 100644
--- a/src/gallium/drivers/panfrost/pan_job.c
+++ b/src/gallium/drivers/panfrost/pan_job.c
@@ -122,7 +122,7 @@ panfrost_batch_cleanup(struct panfrost_context *ctx, struct panfrost_batch *batc
                 panfrost_bo_unreference(bo);
         }
 
-        set_foreach_remove(batch->resources, entry) {
+        set_foreach(batch->resources, entry) {
                 struct panfrost_resource *rsrc = (void *) entry->key;
 
                 if (_mesa_hash_table_search(ctx->writers, rsrc)) {
@@ -364,6 +364,44 @@ panfrost_batch_write_rsrc(struct panfrost_batch *batch,
         panfrost_batch_update_access(batch, rsrc, true);
 }
 
+void
+panfrost_resource_swap_bo(struct panfrost_context *ctx,
+                          struct panfrost_resource *rsrc,
+                          struct panfrost_bo *newbo)
+{
+        /* Any batch writing this resource is writing to the old BO, not the
+         * new BO. After swapping the resource's backing BO, there will be no
+         * writers of the updated resource. Existing writers still hold a
+         * reference to the old BO for reference counting.
+         */
+        struct hash_entry *writer = _mesa_hash_table_search(ctx->writers, rsrc);
+        if (writer) {
+                _mesa_hash_table_remove(ctx->writers, writer);
+                rsrc->track.nr_writers--;
+        }
+
+        /* Likewise, any batch reading this resource is reading the old BO, and
+         * after swapping will not be reading this resource.
+         */
+        unsigned i;
+        foreach_batch(ctx, i) {
+                struct panfrost_batch *batch = &ctx->batches.slots[i];
+                struct set_entry *ent = _mesa_set_search(batch->resources, rsrc);
+
+                if (!ent)
+                        continue;
+
+                _mesa_set_remove(batch->resources, ent);
+                rsrc->track.nr_users--;
+        }
+
+        /* Swap the pointers, dropping a reference to the old BO which is no
+         * long referenced from the resource
+         */
+        panfrost_bo_unreference(rsrc->image.data.bo);
+        rsrc->image.data.bo = newbo;
+}
+
 struct panfrost_bo *
 panfrost_batch_create_bo(struct panfrost_batch *batch, size_t size,
                          uint32_t create_flags, enum pipe_shader_type stage,
diff --git a/src/gallium/drivers/panfrost/pan_job.h b/src/gallium/drivers/panfrost/pan_job.h
index b5c5c270ed8..52eac4e0b17 100644
--- a/src/gallium/drivers/panfrost/pan_job.h
+++ b/src/gallium/drivers/panfrost/pan_job.h
@@ -219,6 +219,11 @@ panfrost_batch_write_rsrc(struct panfrost_batch *batch,
                           struct panfrost_resource *rsrc,
                           enum pipe_shader_type stage);
 
+void
+panfrost_resource_swap_bo(struct panfrost_context *ctx,
+                          struct panfrost_resource *rsrc,
+                          struct panfrost_bo *newbo);
+
 struct panfrost_bo *
 panfrost_batch_create_bo(struct panfrost_batch *batch, size_t size,
                          uint32_t create_flags, enum pipe_shader_type stage,
diff --git a/src/gallium/drivers/panfrost/pan_resource.c b/src/gallium/drivers/panfrost/pan_resource.c
index ac171ca13dc..0bb5e975866 100644
--- a/src/gallium/drivers/panfrost/pan_resource.c
+++ b/src/gallium/drivers/panfrost/pan_resource.c
@@ -1051,6 +1051,14 @@ panfrost_ptr_map(struct pipe_context *pctx,
                 copy_resource = !panfrost_box_covers_resource(resource, box);
         }
 
+        /* Shadowing with separate stencil may require additional accounting.
+         * Bail in these exotic cases.
+         */
+        if (rsrc->separate_stencil) {
+                create_new_bo = false;
+                copy_resource = false;
+        }
+
         if (create_new_bo) {
                 /* Make sure we re-emit any descriptors using this resource */
                 panfrost_dirty_state_all(ctx);
@@ -1078,14 +1086,7 @@ panfrost_ptr_map(struct pipe_context *pctx,
                                 if (copy_resource)
                                         memcpy(newbo->ptr.cpu, rsrc->image.data.bo->ptr.cpu, bo->size);
 
-                                panfrost_bo_unreference(bo);
-                                rsrc->image.data.bo = newbo;
-
-                                /* Swapping out the BO will invalidate batches
-                                 * accessing this resource, flush them but do
-                                 * not wait for them.
-                                 */
-                                panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "Resource shadowing");
+                                panfrost_resource_swap_bo(ctx, rsrc, newbo);
 
 	                        if (!copy_resource &&
                                     drm_is_afbc(rsrc->image.layout.modifier))