diff --git a/src/plugins/renderers/drm/ply-renderer-nouveau-driver.c b/src/plugins/renderers/drm/ply-renderer-nouveau-driver.c
index 33b04b86..b8137dfb 100644
--- a/src/plugins/renderers/drm/ply-renderer-nouveau-driver.c
+++ b/src/plugins/renderers/drm/ply-renderer-nouveau-driver.c
@@ -60,6 +60,8 @@ struct _ply_renderer_buffer
   unsigned long width;
   unsigned long height;
   unsigned long row_stride;
+
+  uint32_t added_fb : 1;
 };
 
 struct _ply_renderer_driver
@@ -250,6 +252,7 @@ create_buffer (ply_renderer_driver_t *driver,
   buffer = ply_renderer_buffer_new (driver,
                                     buffer_object, buffer_id,
                                     width, height, *row_stride);
+  buffer->added_fb = true;
   ply_hashtable_insert (driver->buffers,
                         (void *) (uintptr_t) buffer_id,
                         buffer);
@@ -317,7 +320,8 @@ destroy_buffer (ply_renderer_driver_t *driver,
 
   assert (buffer != NULL);
 
-  drmModeRmFB (driver->device_fd, buffer->id);
+  if (buffer->added_fb)
+    drmModeRmFB (driver->device_fd, buffer->id);
 
   nouveau_bo_ref (NULL, &buffer->object);
 
diff --git a/src/plugins/renderers/drm/ply-renderer-radeon-driver.c b/src/plugins/renderers/drm/ply-renderer-radeon-driver.c
index 0f44ade6..527fbfc9 100644
--- a/src/plugins/renderers/drm/ply-renderer-radeon-driver.c
+++ b/src/plugins/renderers/drm/ply-renderer-radeon-driver.c
@@ -60,6 +60,8 @@ struct _ply_renderer_buffer
   unsigned long width;
   unsigned long height;
   unsigned long row_stride;
+
+  uint32_t added_fb : 1;
 };
 
 struct _ply_renderer_driver
@@ -275,6 +277,7 @@ create_buffer (ply_renderer_driver_t *driver,
   buffer = ply_renderer_buffer_new (driver,
                                     buffer_object, buffer_id,
                                     width, height, *row_stride);
+  buffer->added_fb = true;
   ply_hashtable_insert (driver->buffers,
                         (void *) (uintptr_t) buffer_id,
                         buffer);
@@ -342,7 +345,8 @@ destroy_buffer (ply_renderer_driver_t *driver,
 
   assert (buffer != NULL);
 
-  drmModeRmFB (driver->device_fd, buffer->id);
+  if (buffer->added_fb)
+    drmModeRmFB (driver->device_fd, buffer->id);
 
   radeon_bo_unref (buffer->object);