freedreno: add gmem state cache

Which also has the benefit of getting rid of fd_context::gmem.

Signed-off-by: Rob Clark <robdclark@chromium.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/merge_requests/3503>

src/gallium/drivers/freedreno/freedreno_context.h

@@ -259,14 +259,6 @@ struct fd_context {
 	 */
 	struct pipe_scissor_state disabled_scissor;
 
-	/* Current gmem/tiling configuration.. gets updated on render_tiles()
-	 * if out of date with current maximal-scissor/cpp:
-	 *
-	 * (NOTE: this is kind of related to the batch, but moving it there
-	 * means we'd always have to recalc tiles ever batch)
-	 */
-	struct fd_gmem_stateobj gmem;
-
 	/* Per vsc pipe bo's (a2xx-a5xx): */
 	struct fd_bo *vsc_pipe_bo[32];
 

src/gallium/drivers/freedreno/freedreno_gmem.c

@@ -25,6 +25,7 @@
  */
 
 #include "pipe/p_state.h"
+#include "util/hash_table.h"
 #include "util/u_string.h"
 #include "util/u_memory.h"
 #include "util/u_inlines.h"
@@ -68,6 +69,44 @@
 
 #define BIN_DEBUG 0
 
+/*
+ * GMEM Cache:
+ *
+ * Caches GMEM state based on a given framebuffer state.  The key is
+ * meant to be the minimal set of data that results in a unique gmem
+ * configuration, avoiding multiple keys arriving at the same gmem
+ * state.  For example, the render target format is not part of the
+ * key, only the size per pixel.  And the max_scissor bounds is not
+ * part of they key, only the minx/miny (after clamping to tile
+ * alignment) and width/height.  This ensures that slightly different
+ * max_scissor which would result in the same gmem state, do not
+ * become different keys that map to the same state.
+ */
+
+struct gmem_key {
+	uint16_t minx, miny;
+	uint16_t width, height;
+	uint8_t gmem_page_align;      /* alignment in multiples of 0x1000 to reduce key size */
+	uint8_t nr_cbufs;
+	uint8_t cbuf_cpp[MAX_RENDER_TARGETS];
+	uint8_t zsbuf_cpp[2];
+};
+
+static uint32_t
+gmem_key_hash(const void *_key)
+{
+	const struct gmem_key *key = _key;
+	return _mesa_hash_data(key, sizeof(*key));
+}
+
+static bool
+gmem_key_equals(const void *_a, const void *_b)
+{
+	const struct gmem_key *a = _a;
+	const struct gmem_key *b = _b;
+	return memcmp(a, b, sizeof(*a)) == 0;
+}
+
 static uint32_t bin_width(struct fd_screen *screen)
 {
 	if (is_a4xx(screen) || is_a5xx(screen) || is_a6xx(screen))
@@ -78,148 +117,97 @@ static uint32_t bin_width(struct fd_screen *screen)
 }
 
 static uint32_t
-total_size(uint8_t cbuf_cpp[], uint8_t zsbuf_cpp[2],
+total_size(struct gmem_key *key, uint32_t bin_w, uint32_t bin_h,
-		   uint32_t bin_w, uint32_t bin_h, uint32_t gmem_align,
 		struct fd_gmem_stateobj *gmem)
 {
+	uint32_t gmem_align = key->gmem_page_align * 0x1000;
 	uint32_t total = 0, i;
 
 	for (i = 0; i < MAX_RENDER_TARGETS; i++) {
-		if (cbuf_cpp[i]) {
+		if (key->cbuf_cpp[i]) {
 			gmem->cbuf_base[i] = align(total, gmem_align);
-			total = gmem->cbuf_base[i] + cbuf_cpp[i] * bin_w * bin_h;
+			total = gmem->cbuf_base[i] + key->cbuf_cpp[i] * bin_w * bin_h;
 		}
 	}
 
-	if (zsbuf_cpp[0]) {
+	if (key->zsbuf_cpp[0]) {
 		gmem->zsbuf_base[0] = align(total, gmem_align);
-		total = gmem->zsbuf_base[0] + zsbuf_cpp[0] * bin_w * bin_h;
+		total = gmem->zsbuf_base[0] + key->zsbuf_cpp[0] * bin_w * bin_h;
 	}
 
-	if (zsbuf_cpp[1]) {
+	if (key->zsbuf_cpp[1]) {
 		gmem->zsbuf_base[1] = align(total, gmem_align);
-		total = gmem->zsbuf_base[1] + zsbuf_cpp[1] * bin_w * bin_h;
+		total = gmem->zsbuf_base[1] + key->zsbuf_cpp[1] * bin_w * bin_h;
 	}
 
 	return total;
 }
 
-static void
+static struct fd_gmem_stateobj *
-calculate_tiles(struct fd_batch *batch)
+gmem_stateobj_init(struct fd_screen *screen, struct gmem_key *key)
 {
-	struct fd_context *ctx = batch->ctx;
+	struct fd_gmem_stateobj *gmem =
-	struct fd_screen *screen = ctx->screen;
+			rzalloc(screen->gmem_cache.ht, struct fd_gmem_stateobj);
-	struct fd_gmem_stateobj *gmem = &ctx->gmem;
+	pipe_reference_init(&gmem->reference, 1);
-	struct pipe_scissor_state *scissor = &batch->max_scissor;
+	gmem->screen = screen;
-	struct pipe_framebuffer_state *pfb = &batch->framebuffer;
+	gmem->key = key;
+	list_inithead(&gmem->node);
+
 	const uint32_t gmem_alignw = screen->gmem_alignw;
 	const uint32_t gmem_alignh = screen->gmem_alignh;
 	const unsigned npipes = screen->num_vsc_pipes;
 	const uint32_t gmem_size = screen->gmemsize_bytes;
-	uint32_t minx, miny, width, height;
 	uint32_t nbins_x = 1, nbins_y = 1;
 	uint32_t bin_w, bin_h;
-	uint32_t gmem_align = 0x4000;
 	uint32_t max_width = bin_width(screen);
-	uint8_t cbuf_cpp[MAX_RENDER_TARGETS] = {0}, zsbuf_cpp[2] = {0};
 	uint32_t i, j, t, xoff, yoff;
 	uint32_t tpp_x, tpp_y;
-	bool has_zs = !!(batch->gmem_reason & (FD_GMEM_DEPTH_ENABLED |
-		FD_GMEM_STENCIL_ENABLED | FD_GMEM_CLEARS_DEPTH_STENCIL));
 	int tile_n[npipes];
 
-	if (has_zs) {
+	bin_w = align(key->width, gmem_alignw);
-		struct fd_resource *rsc = fd_resource(pfb->zsbuf->texture);
+	bin_h = align(key->height, gmem_alignh);
-		zsbuf_cpp[0] = rsc->layout.cpp;
-		if (rsc->stencil)
-			zsbuf_cpp[1] = rsc->stencil->layout.cpp;
-	} else {
-		/* we might have a zsbuf, but it isn't used */
-		batch->restore &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL);
-		batch->resolve &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL);
-	}
-	for (i = 0; i < pfb->nr_cbufs; i++) {
-		if (pfb->cbufs[i])
-			cbuf_cpp[i] = util_format_get_blocksize(pfb->cbufs[i]->format);
-		else
-			cbuf_cpp[i] = 4;
-		/* if MSAA, color buffers are super-sampled in GMEM: */
-		cbuf_cpp[i] *= pfb->samples;
-	}
-
-	if (!memcmp(gmem->zsbuf_cpp, zsbuf_cpp, sizeof(zsbuf_cpp)) &&
-		!memcmp(gmem->cbuf_cpp, cbuf_cpp, sizeof(cbuf_cpp)) &&
-		!memcmp(&gmem->scissor, scissor, sizeof(gmem->scissor))) {
-		/* everything is up-to-date */
-		return;
-	}
-
-	if (fd_mesa_debug & FD_DBG_NOSCIS) {
-		minx = 0;
-		miny = 0;
-		width = pfb->width;
-		height = pfb->height;
-	} else {
-		/* round down to multiple of alignment: */
-		minx = scissor->minx & ~(gmem_alignw - 1);
-		miny = scissor->miny & ~(gmem_alignh - 1);
-		width = scissor->maxx - minx;
-		height = scissor->maxy - miny;
-	}
-
-	bin_w = align(width, gmem_alignw);
-	bin_h = align(height, gmem_alignh);
 
 	/* first, find a bin width that satisfies the maximum width
 	 * restrictions:
 	 */
 	while (bin_w > max_width) {
 		nbins_x++;
-		bin_w = align(width / nbins_x, gmem_alignw);
+		bin_w = align(key->width / nbins_x, gmem_alignw);
 	}
 
 	if (fd_mesa_debug & FD_DBG_MSGS) {
 		debug_printf("binning input: cbuf cpp:");
-		for (i = 0; i < pfb->nr_cbufs; i++)
+		for (i = 0; i < key->nr_cbufs; i++)
-			debug_printf(" %d", cbuf_cpp[i]);
+			debug_printf(" %d", key->cbuf_cpp[i]);
 		debug_printf(", zsbuf cpp: %d; %dx%d\n",
-				zsbuf_cpp[0], width, height);
+				key->zsbuf_cpp[0], key->width, key->height);
-	}
-
-	if (is_a20x(screen) && batch->cleared) {
-		/* under normal circumstances the requirement would be 4K
-		 * but the fast clear path requires an alignment of 32K
-		 */
-		gmem_align = 0x8000;
 	}
 
 	/* then find a bin width/height that satisfies the memory
 	 * constraints:
 	 */
-	while (total_size(cbuf_cpp, zsbuf_cpp, bin_w, bin_h, gmem_align, gmem) >
+	while (total_size(key, bin_w, bin_h, gmem) > gmem_size) {
-		   gmem_size) {
 		if (bin_w > bin_h) {
 			nbins_x++;
-			bin_w = align(width / nbins_x, gmem_alignw);
+			bin_w = align(key->width / nbins_x, gmem_alignw);
 		} else {
 			nbins_y++;
-			bin_h = align(height / nbins_y, gmem_alignh);
+			bin_h = align(key->height / nbins_y, gmem_alignh);
 		}
 	}
 
 	DBG("using %d bins of size %dx%d", nbins_x*nbins_y, bin_w, bin_h);
 
-	gmem->scissor = *scissor;
+	memcpy(gmem->cbuf_cpp, key->cbuf_cpp, sizeof(key->cbuf_cpp));
-	memcpy(gmem->cbuf_cpp, cbuf_cpp, sizeof(cbuf_cpp));
+	memcpy(gmem->zsbuf_cpp, key->zsbuf_cpp, sizeof(key->zsbuf_cpp));
-	memcpy(gmem->zsbuf_cpp, zsbuf_cpp, sizeof(zsbuf_cpp));
 	gmem->bin_h = bin_h;
 	gmem->bin_w = bin_w;
 	gmem->nbins_x = nbins_x;
 	gmem->nbins_y = nbins_y;
-	gmem->minx = minx;
+	gmem->minx = key->minx;
-	gmem->miny = miny;
+	gmem->miny = key->miny;
-	gmem->width = width;
+	gmem->width = key->width;
-	gmem->height = height;
+	gmem->height = key->height;
 
 	/*
 	 * Assign tiles and pipes:
@@ -231,7 +219,7 @@ calculate_tiles(struct fd_batch *batch)
 
 #define div_round_up(v, a)  (((v) + (a) - 1) / (a))
 	/* figure out number of tiles per pipe: */
-	if (is_a20x(ctx->screen)) {
+	if (is_a20x(screen)) {
 		/* for a20x we want to minimize the number of "pipes"
 		 * binning data has 3 bits for x/y (8x8) but the edges are used to
 		 * cull off-screen vertices with hw binning, so we have 6x6 pipes
@@ -291,15 +279,15 @@ calculate_tiles(struct fd_batch *batch)
 
 	/* configure tiles: */
 	t = 0;
-	yoff = miny;
+	yoff = key->miny;
 	memset(tile_n, 0, sizeof(tile_n));
 	for (i = 0; i < nbins_y; i++) {
 		uint32_t bw, bh;
 
-		xoff = minx;
+		xoff = key->minx;
 
 		/* clip bin height: */
-		bh = MIN2(bin_h, miny + height - yoff);
+		bh = MIN2(bin_h, key->miny + key->height - yoff);
 
 		for (j = 0; j < nbins_x; j++) {
 			struct fd_tile *tile = &gmem->tile[t];
@@ -312,8 +300,8 @@ calculate_tiles(struct fd_batch *batch)
 			assert(p < gmem->num_vsc_pipes);
 
 			/* clip bin width: */
-			bw = MIN2(bin_w, minx + width - xoff);
+			bw = MIN2(bin_w, key->minx + key->width - xoff);
-			tile->n = !is_a20x(ctx->screen) ? tile_n[p]++ :
+			tile->n = !is_a20x(screen) ? tile_n[p]++ :
 				((i % tpp_y + 1) << 3 | (j % tpp_x + 1));
 			tile->p = p;
 			tile->bin_w = bw;
@@ -344,8 +332,128 @@ calculate_tiles(struct fd_batch *batch)
 			printf("\n");
 		}
 	}
+
+	return gmem;
 }
 
+void
+__fd_gmem_destroy(struct fd_gmem_stateobj *gmem)
+{
+	struct fd_gmem_cache *cache = &gmem->screen->gmem_cache;
+
+	pipe_mutex_assert_locked(gmem->screen->lock);
+
+	_mesa_hash_table_remove_key(cache->ht, gmem->key);
+	list_del(&gmem->node);
+
+	ralloc_free(gmem->key);
+	ralloc_free(gmem);
+}
+
+static struct gmem_key *
+key_init(struct fd_batch *batch)
+{
+	struct fd_screen *screen = batch->ctx->screen;
+	struct pipe_framebuffer_state *pfb = &batch->framebuffer;
+	bool has_zs = pfb->zsbuf && !!(batch->gmem_reason & (FD_GMEM_DEPTH_ENABLED |
+		FD_GMEM_STENCIL_ENABLED | FD_GMEM_CLEARS_DEPTH_STENCIL));
+	struct gmem_key *key = rzalloc(screen->gmem_cache.ht, struct gmem_key);
+
+	if (has_zs) {
+		struct fd_resource *rsc = fd_resource(pfb->zsbuf->texture);
+		key->zsbuf_cpp[0] = rsc->layout.cpp;
+		if (rsc->stencil)
+			key->zsbuf_cpp[1] = rsc->stencil->layout.cpp;
+	} else {
+		/* we might have a zsbuf, but it isn't used */
+		batch->restore &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL);
+		batch->resolve &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL);
+	}
+
+	key->nr_cbufs = pfb->nr_cbufs;
+	for (unsigned i = 0; i < pfb->nr_cbufs; i++) {
+		if (pfb->cbufs[i])
+			key->cbuf_cpp[i] = util_format_get_blocksize(pfb->cbufs[i]->format);
+		else
+			key->cbuf_cpp[i] = 4;
+		/* if MSAA, color buffers are super-sampled in GMEM: */
+		key->cbuf_cpp[i] *= pfb->samples;
+	}
+
+	if (fd_mesa_debug & FD_DBG_NOSCIS) {
+		key->minx = 0;
+		key->miny = 0;
+		key->width = pfb->width;
+		key->height = pfb->height;
+	} else {
+		struct pipe_scissor_state *scissor = &batch->max_scissor;
+
+		/* round down to multiple of alignment: */
+		key->minx = scissor->minx & ~(screen->gmem_alignw - 1);
+		key->miny = scissor->miny & ~(screen->gmem_alignh - 1);
+		key->width = scissor->maxx - key->minx;
+		key->height = scissor->maxy - key->miny;
+	}
+
+	if (is_a20x(screen) && batch->cleared) {
+		/* under normal circumstances the requirement would be 4K
+		 * but the fast clear path requires an alignment of 32K
+		 */
+		key->gmem_page_align = 8;
+	} else {
+		// TODO re-check this across gens.. maybe it should only
+		// be a single page in some cases:
+		key->gmem_page_align = 4;
+	}
+
+	return key;
+}
+
+static struct fd_gmem_stateobj *
+lookup_gmem_state(struct fd_batch *batch)
+{
+	struct fd_screen *screen = batch->ctx->screen;
+	struct fd_gmem_cache *cache = &screen->gmem_cache;
+	struct fd_gmem_stateobj *gmem = NULL;
+	struct gmem_key *key = key_init(batch);
+	uint32_t hash = gmem_key_hash(key);
+
+	mtx_lock(&screen->lock);
+
+	struct hash_entry *entry =
+		_mesa_hash_table_search_pre_hashed(cache->ht, hash, key);
+	if (entry) {
+		ralloc_free(key);
+		goto found;
+	}
+
+	/* limit the # of cached gmem states, discarding the least
+	 * recently used state if needed:
+	 */
+	if (cache->ht->entries >= 20) {
+		struct fd_gmem_stateobj *last =
+			list_last_entry(&cache->lru, struct fd_gmem_stateobj, node);
+		fd_gmem_reference(&last, NULL);
+	}
+
+	entry = _mesa_hash_table_insert_pre_hashed(cache->ht,
+			hash, key, gmem_stateobj_init(screen, key));
+
+found:
+	fd_gmem_reference(&gmem, entry->data);
+	/* Move to the head of the LRU: */
+	list_delinit(&gmem->node);
+	list_add(&gmem->node, &cache->lru);
+
+	mtx_unlock(&screen->lock);
+
+	return gmem;
+}
+
+/*
+ * GMEM render pass
+ */
+
 static void
 render_tiles(struct fd_batch *batch, struct fd_gmem_stateobj *gmem)
 {
@@ -483,9 +591,8 @@ fd_gmem_render_tiles(struct fd_batch *batch)
 		render_sysmem(batch);
 		ctx->stats.batch_sysmem++;
 	} else {
-		struct fd_gmem_stateobj *gmem = &ctx->gmem;
+		struct fd_gmem_stateobj *gmem = lookup_gmem_state(batch);
 		batch->gmem_state = gmem;
-		calculate_tiles(batch);
 		DBG("%p: rendering %dx%d tiles %ux%u (%s/%s)",
 			batch, pfb->width, pfb->height, gmem->nbins_x, gmem->nbins_y,
 			util_format_short_name(pipe_surface_format(pfb->cbufs[0])),
@@ -493,6 +600,12 @@ fd_gmem_render_tiles(struct fd_batch *batch)
 		if (ctx->query_prepare)
 			ctx->query_prepare(batch, gmem->nbins_x * gmem->nbins_y);
 		render_tiles(batch, gmem);
+		batch->gmem_state = NULL;
+
+		mtx_lock(&ctx->screen->lock);
+		fd_gmem_reference(&gmem, NULL);
+		mtx_unlock(&ctx->screen->lock);
+
 		ctx->stats.batch_gmem++;
 	}
 
@@ -513,3 +626,20 @@ fd_gmem_needs_restore(struct fd_batch *batch, const struct fd_tile *tile,
 
 	return true;
 }
+
+void
+fd_gmem_screen_init(struct pipe_screen *pscreen)
+{
+	struct fd_gmem_cache *cache = &fd_screen(pscreen)->gmem_cache;
+
+	cache->ht = _mesa_hash_table_create(NULL, gmem_key_hash, gmem_key_equals);
+	list_inithead(&cache->lru);
+}
+
+void
+fd_gmem_screen_fini(struct pipe_screen *pscreen)
+{
+	struct fd_gmem_cache *cache = &fd_screen(pscreen)->gmem_cache;
+
+	_mesa_hash_table_destroy(cache->ht, NULL);
+}

src/gallium/drivers/freedreno/freedreno_gmem.h

@@ -27,7 +27,8 @@
 #ifndef FREEDRENO_GMEM_H_
 #define FREEDRENO_GMEM_H_
 
-#include "pipe/p_context.h"
+#include "pipe/p_state.h"
+#include "util/list.h"
 
 #include "freedreno_util.h"
 
@@ -45,7 +46,10 @@ struct fd_tile {
 };
 
 struct fd_gmem_stateobj {
-	struct pipe_scissor_state scissor;
+	struct pipe_reference reference;
+	struct fd_screen *screen;
+	void *key;
+
 	uint32_t cbuf_base[MAX_RENDER_TARGETS];
 	uint32_t zsbuf_base[2];
 	uint8_t cbuf_cpp[MAX_RENDER_TARGETS];
@@ -59,6 +63,26 @@ struct fd_gmem_stateobj {
 
 	struct fd_vsc_pipe vsc_pipe[32];
 	struct fd_tile     tile[512];
+
+	struct list_head node;
+};
+
+void __fd_gmem_destroy(struct fd_gmem_stateobj *gmem);
+
+static inline void
+fd_gmem_reference(struct fd_gmem_stateobj **ptr, struct fd_gmem_stateobj *gmem)
+{
+	struct fd_gmem_stateobj *old_gmem = *ptr;
+
+	if (pipe_reference(&(*ptr)->reference, &gmem->reference))
+		__fd_gmem_destroy(old_gmem);
+
+	*ptr = gmem;
+}
+
+struct fd_gmem_cache {
+	struct hash_table *ht;
+	struct list_head lru;
 };
 
 struct fd_batch;
@@ -68,4 +92,8 @@ void fd_gmem_render_tiles(struct fd_batch *batch);
 bool fd_gmem_needs_restore(struct fd_batch *batch, const struct fd_tile *tile,
 		uint32_t buffers);
 
+struct pipe_screen;
+void fd_gmem_screen_init(struct pipe_screen *pscreen);
+void fd_gmem_screen_fini(struct pipe_screen *pscreen);
+
 #endif /* FREEDRENO_GMEM_H_ */

src/gallium/drivers/freedreno/freedreno_screen.c

@@ -154,6 +154,7 @@ fd_screen_destroy(struct pipe_screen *pscreen)
 		FREE(screen->ro);
 
 	fd_bc_fini(&screen->batch_cache);
+	fd_gmem_screen_fini(pscreen);
 
 	slab_destroy_parent(&screen->transfer_pool);
 
@@ -969,6 +970,7 @@ fd_screen_create(struct fd_device *dev, struct renderonly *ro)
 
 	fd_resource_screen_init(pscreen);
 	fd_query_screen_init(pscreen);
+	fd_gmem_screen_init(pscreen);
 
 	pscreen->get_name = fd_screen_get_name;
 	pscreen->get_vendor = fd_screen_get_vendor;

src/gallium/drivers/freedreno/freedreno_screen.h

@@ -38,6 +38,7 @@
 #include "renderonly/renderonly.h"
 
 #include "freedreno_batch_cache.h"
+#include "freedreno_gmem.h"
 #include "freedreno_util.h"
 
 struct fd_bo;
@@ -113,6 +114,7 @@ struct fd_screen {
 	int64_t cpu_gpu_time_delta;
 
 	struct fd_batch_cache batch_cache;
+	struct fd_gmem_cache gmem_cache;
 
 	bool reorder;