diff --git a/src/gst/gstpipewirepool.c b/src/gst/gstpipewirepool.c
index 0dc9ef0a2..547670d1c 100644
--- a/src/gst/gstpipewirepool.c
+++ b/src/gst/gstpipewirepool.c
@@ -119,6 +119,21 @@ void gst_pipewire_pool_wrap_buffer (GstPipeWirePool *pool, struct pw_buffer *b)
   b->user_data = data;
 }
 
+void gst_pipewire_pool_remove_buffer (GstPipeWirePool *pool, struct pw_buffer *b)
+{
+  GstPipeWirePoolData *data = b->user_data;
+
+  data->b = NULL;
+  data->header = NULL;
+  data->crop = NULL;
+  data->videotransform = NULL;
+
+  gst_buffer_remove_all_memory (data->buf);
+
+  /* this will also destroy the pool data, if this is the last reference */
+  gst_clear_buffer (&data->buf);
+}
+
 GstPipeWirePoolData *gst_pipewire_pool_get_data (GstBuffer *buffer)
 {
   return gst_mini_object_get_qdata (GST_MINI_OBJECT_CAST (buffer), pool_data_quark);
diff --git a/src/gst/gstpipewirepool.h b/src/gst/gstpipewirepool.h
index b6944b4b1..cceb8833b 100644
--- a/src/gst/gstpipewirepool.h
+++ b/src/gst/gstpipewirepool.h
@@ -66,6 +66,7 @@ GType gst_pipewire_pool_get_type (void);
 GstPipeWirePool *  gst_pipewire_pool_new           (void);
 
 void gst_pipewire_pool_wrap_buffer (GstPipeWirePool *pool, struct pw_buffer *buffer);
+void gst_pipewire_pool_remove_buffer (GstPipeWirePool *pool, struct pw_buffer *buffer);
 
 GstPipeWirePoolData *gst_pipewire_pool_get_data (GstBuffer *buffer);
 
diff --git a/src/gst/gstpipewiresink.c b/src/gst/gstpipewiresink.c
index a90412c17..1ea127a4d 100644
--- a/src/gst/gstpipewiresink.c
+++ b/src/gst/gstpipewiresink.c
@@ -472,11 +472,8 @@ static void
 on_remove_buffer (void *_data, struct pw_buffer *b)
 {
   GstPipeWireSink *pwsink = _data;
-  GstPipeWirePoolData *data = b->user_data;
-
-  GST_LOG_OBJECT (pwsink, "remove buffer");
-
-  gst_buffer_unref (data->buf);
+  GST_DEBUG_OBJECT (pwsink, "remove pw_buffer %p", b);
+  gst_pipewire_pool_remove_buffer (pwsink->pool, b);
 }
 
 static void