r600: Fix userspace pointer support for evergreen compute.

Resources returned by r600_buffer_from_user_memory() are not compatible
with the evergreen compute memory pool, though they're added to it anyway.

This results in a segfault reproducible from Clover when the user passes
CL_MEM_USE_HOST_PTR.

This patch allows user_ptr resources to participate in the compute global
memory pool as intended. The result appears to finally allow for zero-copy
DMA out of userspace for anonymous pages.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/16114>

src/gallium/drivers/r600/compute_memory_pool.c

@@ -350,7 +350,7 @@ static int compute_memory_promote_item(struct compute_memory_pool *pool,
 		 * In this case, we need to keep the temporary buffer 'alive'
 		 * because it is possible to keep a map active for reading
 		 * while a kernel (that reads from it) executes */
-		if (!(item->status & ITEM_MAPPED_FOR_READING)) {
+		if (!(item->status & ITEM_MAPPED_FOR_READING) && !is_item_user_ptr(item)) {
 			pool->screen->b.b.resource_destroy(screen, src);
 			item->real_buffer = NULL;
 		}
@@ -494,6 +494,22 @@ static void compute_memory_move_item(struct compute_memory_pool *pool,
 	item->start_in_dw = new_start_in_dw;
 }
 
+/**
+ * Frees one item for compute_memory_free()
+ */
+static void compute_memory_free_item(struct pipe_screen *screen,
+	struct compute_memory_item *item)
+{
+	struct pipe_resource *res = (struct pipe_resource *)item->real_buffer;
+
+	list_del(&item->link);
+
+	if (res && !is_item_user_ptr(item))
+		screen->resource_destroy(screen, res);
+
+	free(item);
+}
+
 /**
  * Frees the memory associated to the item with id \a id from the pool.
  * \param id	The id of the item to be freed.
@@ -502,45 +518,23 @@ void compute_memory_free(struct compute_memory_pool* pool, int64_t id)
 {
 	struct compute_memory_item *item, *next;
 	struct pipe_screen *screen = (struct pipe_screen *)pool->screen;
-	struct pipe_resource *res;
 
 	COMPUTE_DBG(pool->screen, "* compute_memory_free() id + %"PRIi64" \n", id);
 
 	LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->item_list, link) {
-
 		if (item->id == id) {
-
 			if (item->link.next != pool->item_list) {
 				pool->status |= POOL_FRAGMENTED;
 			}
 
-			list_del(&item->link);
-
-			if (item->real_buffer) {
-				res = (struct pipe_resource *)item->real_buffer;
-				pool->screen->b.b.resource_destroy(
-						screen, res);
-			}
-
-			free(item);
-
+			compute_memory_free_item(screen, item);
 			return;
 		}
 	}
 
 	LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->unallocated_list, link) {
-
 		if (item->id == id) {
-			list_del(&item->link);
-
-			if (item->real_buffer) {
-				res = (struct pipe_resource *)item->real_buffer;
-				pool->screen->b.b.resource_destroy(
-						screen, res);
-			}
-
-			free(item);
-
+			compute_memory_free_item(screen, item);
 			return;
 		}
 	}

src/gallium/drivers/r600/compute_memory_pool.h

@@ -82,6 +82,12 @@ static inline int is_item_in_pool(struct compute_memory_item *item)
 	return item->start_in_dw != -1;
 }
 
+static inline int is_item_user_ptr(struct compute_memory_item *item)
+{
+	assert(item->real_buffer);
+	return item->real_buffer->b.is_user_ptr;
+}
+
 struct compute_memory_pool* compute_memory_pool_new(struct r600_screen *rscreen);
 
 void compute_memory_pool_delete(struct compute_memory_pool* pool);

src/gallium/drivers/r600/evergreen_compute.c

@@ -1277,6 +1277,9 @@ void *r600_compute_global_transfer_map(struct pipe_context *ctx,
 	assert(box->y == 0);
 	assert(box->z == 0);
 
+	if (buffer->base.b.is_user_ptr)
+		return NULL;
+
 	///TODO: do it better, mapping is not possible if the pool is too big
 	return pipe_buffer_map_range(ctx, dst,
 			offset, box->width, usage, ptransfer);
@@ -1311,9 +1314,12 @@ void r600_compute_global_buffer_destroy(struct pipe_screen *screen,
 	rscreen = (struct r600_screen*)screen;
 
 	compute_memory_free(rscreen->global_pool, buffer->chunk->id);
-
 	buffer->chunk = NULL;
-	free(res);
+
+	if (buffer->base.b.is_user_ptr)
+		r600_buffer_destroy(screen, res);
+	else
+		free(res);
 }
 
 struct pipe_resource *r600_compute_global_buffer_create(struct pipe_screen *screen,

src/gallium/drivers/r600/r600_buffer_common.c

@@ -26,6 +26,7 @@
 
 #include "r600_cs.h"
 #include "evergreen_compute.h"
+#include "compute_memory_pool.h"
 #include "util/u_memory.h"
 #include "util/u_upload_mgr.h"
 #include <inttypes.h>
@@ -347,7 +348,8 @@ void *r600_buffer_transfer_map(struct pipe_context *ctx,
 	uint8_t *data;
 
 	if (r600_resource(resource)->compute_global_bo) {
-		return r600_compute_global_transfer_map(ctx, resource, level, usage, box, ptransfer);
+		if ((data = r600_compute_global_transfer_map(ctx, resource, level, usage, box, ptransfer)))
+			return data;
 	}
 
 	assert(box->x + box->width <= resource->width0);
@@ -524,8 +526,9 @@ void r600_buffer_transfer_unmap(struct pipe_context *ctx,
 {
 	struct r600_common_context *rctx = (struct r600_common_context*)ctx;
 	struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
+	struct r600_resource *rtransferr = r600_resource(transfer->resource);
 
-	if (r600_resource(transfer->resource)->compute_global_bo) {
+	if (rtransferr->compute_global_bo && !rtransferr->b.is_user_ptr) {
 		r600_compute_global_transfer_unmap(ctx, transfer);
 		return;
 	}
@@ -636,7 +639,15 @@ r600_buffer_from_user_memory(struct pipe_screen *screen,
 {
 	struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
 	struct radeon_winsys *ws = rscreen->ws;
-	struct r600_resource *rbuffer = r600_alloc_buffer_struct(screen, templ);
+	struct r600_resource *rbuffer;
+
+	if ((templ->bind & PIPE_BIND_GLOBAL) &&
+	    (templ->bind & PIPE_BIND_COMPUTE_RESOURCE)) {
+		rbuffer = r600_resource(r600_compute_global_buffer_create(screen, templ));
+		((struct r600_resource_global *)rbuffer)->chunk->real_buffer = rbuffer;
+	} else {
+		rbuffer = r600_alloc_buffer_struct(screen, templ);
+	}
 
 	rbuffer->domains = RADEON_DOMAIN_GTT;
 	rbuffer->flags = 0;