diff --git a/src/gallium/drivers/panfrost/pan_resource.c b/src/gallium/drivers/panfrost/pan_resource.c
index c96546398f1..4727483e44f 100644
--- a/src/gallium/drivers/panfrost/pan_resource.c
+++ b/src/gallium/drivers/panfrost/pan_resource.c
@@ -1044,8 +1044,6 @@ panfrost_ptr_unmap(struct pipe_context *pctx,
                                 assert(transfer->box.depth == 1);
 
                                 if (panfrost_should_linear_convert(prsrc, transfer)) {
-                                        prsrc->modifier = DRM_FORMAT_MOD_LINEAR;
-
                                         util_copy_rect(
                                                 bo->ptr.cpu + prsrc->slices[0].offset,
                                                 prsrc->base.format,
@@ -1056,6 +1054,8 @@ panfrost_ptr_unmap(struct pipe_context *pctx,
                                                 trans->map,
                                                 transfer->stride,
                                                 0, 0);
+
+                                        panfrost_resource_setup(dev, prsrc, NULL, DRM_FORMAT_MOD_LINEAR);
                                 } else {
                                         panfrost_store_tiled_image(
                                                 bo->ptr.cpu + prsrc->slices[transfer->level].offset,