diff --git a/src/gallium/drivers/panfrost/Makefile.sources b/src/gallium/drivers/panfrost/Makefile.sources index ce613d6793f..470dfb31efe 100644 --- a/src/gallium/drivers/panfrost/Makefile.sources +++ b/src/gallium/drivers/panfrost/Makefile.sources @@ -8,13 +8,10 @@ C_SOURCES := \ pan_compute.c \ pan_context.c \ pan_context.h \ - pan_fragment.c \ pan_job.c \ pan_job.h \ - pan_mfbd.c \ pan_public.h \ pan_resource.c \ pan_resource.h \ pan_screen.c \ - pan_screen.h \ - pan_sfbd.c \ + pan_screen.h diff --git a/src/gallium/drivers/panfrost/ci/deqp-panfrost-t720-fails.txt b/src/gallium/drivers/panfrost/ci/deqp-panfrost-t720-fails.txt index 97c764f4ea4..fce4eb729fd 100644 --- a/src/gallium/drivers/panfrost/ci/deqp-panfrost-t720-fails.txt +++ b/src/gallium/drivers/panfrost/ci/deqp-panfrost-t720-fails.txt @@ -1,32 +1,3 @@ -dEQP-GLES2.functional.depth_stencil_clear.depth_stencil_masked,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgb565_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgb565_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgb5_a1_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgb5_a1_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgba4_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgba4_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_tex2d_rgba_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_tex2d_rgba_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_tex2d_rgb_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_tex2d_rgb_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgb565_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgb565_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgb5_a1_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgb5_a1_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgba4_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgba4_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_tex2d_rgba_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_tex2d_rgba_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_tex2d_rgb_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_tex2d_rgb_stencil_index8,Fail -dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgb5_a1_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgba4_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.shared_colorbuffer.tex2d_rgb_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgb565_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgb5_a1_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgba4_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.shared_depthbuffer.tex2d_rgba_depth_component16,Fail -dEQP-GLES2.functional.fbo.render.shared_depthbuffer.tex2d_rgb_depth_component16,Fail dEQP-GLES2.functional.fragment_ops.blend.equation_src_func_dst_func.add_src_color_one_minus_dst_alpha,Fail dEQP-GLES2.functional.fragment_ops.blend.equation_src_func_dst_func.reverse_subtract_one_minus_dst_alpha_one_minus_src_alpha,Fail dEQP-GLES2.functional.fragment_ops.blend.equation_src_func_dst_func.reverse_subtract_one_minus_dst_color_one_minus_src_alpha,Fail diff --git a/src/gallium/drivers/panfrost/meson.build b/src/gallium/drivers/panfrost/meson.build index ccee3ce00d2..5afea1ad7fc 100644 --- a/src/gallium/drivers/panfrost/meson.build +++ b/src/gallium/drivers/panfrost/meson.build @@ -32,9 +32,6 @@ files_panfrost = files( 'pan_blend_cso.c', 'pan_cmdstream.c', 'pan_compute.c', - 'pan_fragment.c', - 'pan_sfbd.c', - 'pan_mfbd.c', ) panfrost_includes = [ diff --git a/src/gallium/drivers/panfrost/pan_cmdstream.c b/src/gallium/drivers/panfrost/pan_cmdstream.c index 3c4c5e4ae21..6bea630e9ef 100644 --- a/src/gallium/drivers/panfrost/pan_cmdstream.c +++ b/src/gallium/drivers/panfrost/pan_cmdstream.c @@ -2251,3 +2251,117 @@ panfrost_emit_vertex_tiler_jobs(struct panfrost_batch *batch, MALI_JOB_TYPE_TILER, false, false, vertex, 0, tiler_job, false); } + +void +panfrost_emit_tls(struct panfrost_batch *batch) +{ + struct panfrost_device *dev = pan_device(batch->ctx->base.screen); + + /* Emitted with the FB descriptor on Midgard. */ + if (!pan_is_bifrost(dev)) + return; + + struct panfrost_bo *tls_bo = + batch->stack_size ? + panfrost_batch_get_scratchpad(batch, + batch->stack_size, + dev->thread_tls_alloc, + dev->core_count): + NULL; + struct pan_tls_info tls = { + .tls = { + .ptr = tls_bo ? tls_bo->ptr.gpu : 0, + .size = batch->stack_size, + }, + }; + + assert(batch->tls.cpu); + pan_emit_tls(dev, &tls, batch->tls.cpu); +} + +void +panfrost_emit_fbd(struct panfrost_batch *batch, + const struct pan_fb_info *fb) +{ + struct panfrost_device *dev = pan_device(batch->ctx->base.screen); + struct panfrost_bo *tls_bo = + batch->stack_size ? + panfrost_batch_get_scratchpad(batch, + batch->stack_size, + dev->thread_tls_alloc, + dev->core_count): + NULL; + struct pan_tls_info tls = { + .tls = { + .ptr = tls_bo ? tls_bo->ptr.gpu : 0, + .size = batch->stack_size, + }, + }; + + batch->framebuffer.gpu |= + pan_emit_fbd(dev, fb, &tls, &batch->tiler_ctx, + batch->framebuffer.cpu); +} + +/* Mark a surface as written */ + +static void +panfrost_initialize_surface(struct panfrost_batch *batch, + struct pipe_surface *surf) +{ + if (!surf) + return; + + unsigned level = surf->u.tex.level; + struct panfrost_resource *rsrc = pan_resource(surf->texture); + + rsrc->state.slices[level].data_valid = true; +} + +/* Generate a fragment job. This should be called once per frame. (According to + * presentations, this is supposed to correspond to eglSwapBuffers) */ + +mali_ptr +panfrost_emit_fragment_job(struct panfrost_batch *batch, + const struct pan_fb_info *pfb) +{ + struct panfrost_device *dev = pan_device(batch->ctx->base.screen); + + /* Mark the affected buffers as initialized, since we're writing to it. + * Also, add the surfaces we're writing to to the batch */ + + struct pipe_framebuffer_state *fb = &batch->key; + + for (unsigned i = 0; i < fb->nr_cbufs; ++i) + panfrost_initialize_surface(batch, fb->cbufs[i]); + + panfrost_initialize_surface(batch, fb->zsbuf); + + /* The passed tile coords can be out of range in some cases, so we need + * to clamp them to the framebuffer size to avoid a TILE_RANGE_FAULT. + * Theoretically we also need to clamp the coordinates positive, but we + * avoid that edge case as all four values are unsigned. Also, + * theoretically we could clamp the minima, but if that has to happen + * the asserts would fail anyway (since the maxima would get clamped + * and then be smaller than the minima). An edge case of sorts occurs + * when no scissors are added to draw, so by default min=~0 and max=0. + * But that can't happen if any actual drawing occurs (beyond a + * wallpaper reload), so this is again irrelevant in practice. */ + + batch->maxx = MIN2(batch->maxx, fb->width); + batch->maxy = MIN2(batch->maxy, fb->height); + + /* Rendering region must be at least 1x1; otherwise, there is nothing + * to do and the whole job chain should have been discarded. */ + + assert(batch->maxx > batch->minx); + assert(batch->maxy > batch->miny); + + struct panfrost_ptr transfer = + panfrost_pool_alloc_desc(&batch->pool, FRAGMENT_JOB); + + pan_emit_fragment_job(dev, pfb, batch->framebuffer.gpu, + transfer.cpu); + + return transfer.gpu; +} diff --git a/src/gallium/drivers/panfrost/pan_cmdstream.h b/src/gallium/drivers/panfrost/pan_cmdstream.h index 4a46976a247..b0ac3bb25ec 100644 --- a/src/gallium/drivers/panfrost/pan_cmdstream.h +++ b/src/gallium/drivers/panfrost/pan_cmdstream.h @@ -92,6 +92,17 @@ panfrost_emit_vertex_tiler_jobs(struct panfrost_batch *batch, const struct panfrost_ptr *vertex_job, const struct panfrost_ptr *tiler_job); +mali_ptr +panfrost_emit_fragment_job(struct panfrost_batch *batch, + const struct pan_fb_info *fb); + +void +panfrost_emit_tls(struct panfrost_batch *batch); + +void +panfrost_emit_fbd(struct panfrost_batch *batch, + const struct pan_fb_info *fb); + static inline unsigned panfrost_translate_compare_func(enum pipe_compare_func in) { diff --git a/src/gallium/drivers/panfrost/pan_context.c b/src/gallium/drivers/panfrost/pan_context.c index d8c8026aa52..c158cb482ea 100644 --- a/src/gallium/drivers/panfrost/pan_context.c +++ b/src/gallium/drivers/panfrost/pan_context.c @@ -57,69 +57,6 @@ #include "decode.h" #include "util/pan_lower_framebuffer.h" -void -panfrost_emit_midg_tiler(struct panfrost_batch *batch, - struct mali_midgard_tiler_packed *tp, - unsigned vertex_count) -{ - struct panfrost_device *device = pan_device(batch->ctx->base.screen); - bool hierarchy = !(device->quirks & MIDGARD_NO_HIER_TILING); - unsigned height = batch->key.height; - unsigned width = batch->key.width; - - pan_pack(tp, MIDGARD_TILER, t) { - t.hierarchy_mask = - panfrost_choose_hierarchy_mask(width, height, - vertex_count, hierarchy); - - /* Compute the polygon header size and use that to offset the body */ - - unsigned header_size = - panfrost_tiler_header_size(width, height, - t.hierarchy_mask, hierarchy); - - t.polygon_list_size = - panfrost_tiler_full_size(width, height, t.hierarchy_mask, - hierarchy); - - if (vertex_count) { - t.polygon_list = - panfrost_batch_get_polygon_list(batch, - header_size + - t.polygon_list_size); - - t.heap_start = device->tiler_heap->ptr.gpu; - t.heap_end = device->tiler_heap->ptr.gpu + - device->tiler_heap->size; - } else { - struct panfrost_bo *tiler_dummy; - - tiler_dummy = panfrost_batch_get_tiler_dummy(batch); - header_size = MALI_MIDGARD_TILER_MINIMUM_HEADER_SIZE; - - /* The tiler is disabled, so don't allow the tiler heap */ - t.heap_start = tiler_dummy->ptr.gpu; - t.heap_end = t.heap_start; - - /* Use a dummy polygon list */ - t.polygon_list = tiler_dummy->ptr.gpu; - - /* Disable the tiler */ - if (hierarchy) - t.hierarchy_mask |= MALI_MIDGARD_TILER_DISABLED; - else { - t.hierarchy_mask = MALI_MIDGARD_TILER_USER; - t.polygon_list_size = MALI_MIDGARD_TILER_MINIMUM_HEADER_SIZE + 4; - - /* We don't have a WRITE_VALUE job, so write the polygon list manually */ - uint32_t *polygon_list_body = (uint32_t *) (tiler_dummy->ptr.cpu + header_size); - polygon_list_body[0] = 0xa0000000; /* TODO: Just that? */ - } - } - t.polygon_list_body = t.polygon_list + header_size; - } -} - static void panfrost_clear( struct pipe_context *pipe, @@ -523,7 +460,7 @@ panfrost_direct_draw(struct panfrost_context *ctx, unsigned vertex_count = ctx->vertex_count; - mali_ptr shared_mem = panfrost_batch_reserve_framebuffer(batch); + mali_ptr shared_mem = panfrost_batch_reserve_tls(batch); unsigned min_index = 0, max_index = 0; mali_ptr indices = 0; diff --git a/src/gallium/drivers/panfrost/pan_context.h b/src/gallium/drivers/panfrost/pan_context.h index 2450ec34c04..8194d6d25f3 100644 --- a/src/gallium/drivers/panfrost/pan_context.h +++ b/src/gallium/drivers/panfrost/pan_context.h @@ -315,23 +315,6 @@ panfrost_flush( bool panfrost_render_condition_check(struct panfrost_context *ctx); -mali_ptr panfrost_sfbd_fragment(struct panfrost_batch *batch, bool has_draws); -mali_ptr panfrost_mfbd_fragment(struct panfrost_batch *batch, bool has_draws); - -void -panfrost_attach_mfbd(struct panfrost_batch *batch, unsigned vertex_count); - -void -panfrost_attach_sfbd(struct panfrost_batch *batch, unsigned vertex_count); - -void -panfrost_emit_midg_tiler(struct panfrost_batch *batch, - struct mali_midgard_tiler_packed *tp, - unsigned vertex_count); - -mali_ptr -panfrost_fragment_job(struct panfrost_batch *batch, bool has_draws); - void panfrost_shader_compile(struct panfrost_context *ctx, enum pipe_shader_ir ir_type, diff --git a/src/gallium/drivers/panfrost/pan_fragment.c b/src/gallium/drivers/panfrost/pan_fragment.c deleted file mode 100644 index a36c6c0c8af..00000000000 --- a/src/gallium/drivers/panfrost/pan_fragment.c +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2019-2020 Collabora, Ltd. - * Copyright 2018-2019 Alyssa Rosenzweig - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include "pan_context.h" -#include "pan_util.h" -#include "panfrost-quirks.h" - -#include "util/format/u_format.h" - -/* Mark a surface as written */ - -static void -panfrost_initialize_surface( - struct panfrost_batch *batch, - struct pipe_surface *surf) -{ - if (!surf) - return; - - unsigned level = surf->u.tex.level; - struct panfrost_resource *rsrc = pan_resource(surf->texture); - - rsrc->state.slices[level].data_valid = true; -} - -/* Generate a fragment job. This should be called once per frame. (According to - * presentations, this is supposed to correspond to eglSwapBuffers) */ - -mali_ptr -panfrost_fragment_job(struct panfrost_batch *batch, bool has_draws) -{ - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - - mali_ptr framebuffer = (dev->quirks & MIDGARD_SFBD) ? - panfrost_sfbd_fragment(batch, has_draws) : - panfrost_mfbd_fragment(batch, has_draws); - - /* Mark the affected buffers as initialized, since we're writing to it. - * Also, add the surfaces we're writing to to the batch */ - - struct pipe_framebuffer_state *fb = &batch->key; - - for (unsigned i = 0; i < fb->nr_cbufs; ++i) - panfrost_initialize_surface(batch, fb->cbufs[i]); - - panfrost_initialize_surface(batch, fb->zsbuf); - - /* The passed tile coords can be out of range in some cases, so we need - * to clamp them to the framebuffer size to avoid a TILE_RANGE_FAULT. - * Theoretically we also need to clamp the coordinates positive, but we - * avoid that edge case as all four values are unsigned. Also, - * theoretically we could clamp the minima, but if that has to happen - * the asserts would fail anyway (since the maxima would get clamped - * and then be smaller than the minima). An edge case of sorts occurs - * when no scissors are added to draw, so by default min=~0 and max=0. - * But that can't happen if any actual drawing occurs (beyond a - * wallpaper reload), so this is again irrelevant in practice. */ - - batch->maxx = MIN2(batch->maxx, fb->width); - batch->maxy = MIN2(batch->maxy, fb->height); - - /* Rendering region must be at least 1x1; otherwise, there is nothing - * to do and the whole job chain should have been discarded. */ - - assert(batch->maxx > batch->minx); - assert(batch->maxy > batch->miny); - - struct panfrost_ptr transfer = - panfrost_pool_alloc_desc(&batch->pool, FRAGMENT_JOB); - - pan_section_pack(transfer.cpu, FRAGMENT_JOB, HEADER, header) { - header.type = MALI_JOB_TYPE_FRAGMENT; - header.index = 1; - } - - pan_section_pack(transfer.cpu, FRAGMENT_JOB, PAYLOAD, payload) { - payload.bound_min_x = batch->minx >> MALI_TILE_SHIFT; - payload.bound_min_y = batch->miny >> MALI_TILE_SHIFT; - - /* Batch max values are inclusive, we need to subtract 1. */ - payload.bound_max_x = (batch->maxx - 1) >> MALI_TILE_SHIFT; - payload.bound_max_y = (batch->maxy - 1) >> MALI_TILE_SHIFT; - payload.framebuffer = framebuffer; - } - - return transfer.gpu; -} diff --git a/src/gallium/drivers/panfrost/pan_job.c b/src/gallium/drivers/panfrost/pan_job.c index 817ee6a9896..ab7d7a2fa31 100644 --- a/src/gallium/drivers/panfrost/pan_job.c +++ b/src/gallium/drivers/panfrost/pan_job.c @@ -29,6 +29,7 @@ #include "drm-uapi/panfrost_drm.h" #include "pan_bo.h" +#include "pan_blitter.h" #include "pan_context.h" #include "util/hash_table.h" #include "util/ralloc.h" @@ -614,24 +615,47 @@ panfrost_batch_create_bo(struct panfrost_batch *batch, size_t size, * the polygon list. It's perfectly fast to use allocate/free BO directly, * since we'll hit the BO cache and this is one-per-batch anyway. */ -mali_ptr -panfrost_batch_get_polygon_list(struct panfrost_batch *batch, unsigned size) +static mali_ptr +panfrost_batch_get_polygon_list(struct panfrost_batch *batch) { - if (batch->polygon_list) { - assert(batch->polygon_list->size >= size); - } else { - /* Create the BO as invisible, as there's no reason to map */ + struct panfrost_device *dev = pan_device(batch->ctx->base.screen); + + assert(!pan_is_bifrost(dev)); + + if (!batch->tiler_ctx.midgard.polygon_list) { + bool has_draws = batch->scoreboard.first_tiler != NULL; + unsigned size = + panfrost_tiler_get_polygon_list_size(dev, + batch->key.width, + batch->key.height, + has_draws); size = util_next_power_of_two(size); - batch->polygon_list = panfrost_batch_create_bo(batch, size, - PAN_BO_INVISIBLE, - PAN_BO_ACCESS_PRIVATE | - PAN_BO_ACCESS_RW | - PAN_BO_ACCESS_VERTEX_TILER | - PAN_BO_ACCESS_FRAGMENT); + /* Create the BO as invisible if we can. In the non-hierarchical tiler case, + * we need to write the polygon list manually because there's not WRITE_VALUE + * job in the chain (maybe we should add one...). */ + bool init_polygon_list = !has_draws && (dev->quirks & MIDGARD_NO_HIER_TILING); + batch->tiler_ctx.midgard.polygon_list = + panfrost_batch_create_bo(batch, size, + init_polygon_list ? 0 : PAN_BO_INVISIBLE, + PAN_BO_ACCESS_PRIVATE | + PAN_BO_ACCESS_RW | + PAN_BO_ACCESS_VERTEX_TILER | + PAN_BO_ACCESS_FRAGMENT); + + + if (init_polygon_list) { + assert(batch->tiler_ctx.midgard.polygon_list->ptr.cpu); + uint32_t *polygon_list_body = + batch->tiler_ctx.midgard.polygon_list->ptr.cpu + + MALI_MIDGARD_TILER_MINIMUM_HEADER_SIZE; + polygon_list_body[0] = 0xa0000000; /* TODO: Just that? */ + } + + batch->tiler_ctx.midgard.disable = !has_draws; } - return batch->polygon_list->ptr.gpu; + return batch->tiler_ctx.midgard.polygon_list->ptr.gpu; } struct panfrost_bo * @@ -679,62 +703,174 @@ panfrost_batch_get_shared_memory(struct panfrost_batch *batch, mali_ptr panfrost_batch_get_bifrost_tiler(struct panfrost_batch *batch, unsigned vertex_count) { + struct panfrost_device *dev = pan_device(batch->ctx->base.screen); + assert(pan_is_bifrost(dev)); + if (!vertex_count) return 0; - if (batch->tiler_meta) - return batch->tiler_meta; + if (batch->tiler_ctx.bifrost) + return batch->tiler_ctx.bifrost; - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); struct panfrost_ptr t = panfrost_pool_alloc_desc(&batch->pool, BIFROST_TILER_HEAP); - pan_pack(t.cpu, BIFROST_TILER_HEAP, heap) { - heap.size = dev->tiler_heap->size; - heap.base = dev->tiler_heap->ptr.gpu; - heap.bottom = dev->tiler_heap->ptr.gpu; - heap.top = dev->tiler_heap->ptr.gpu + dev->tiler_heap->size; - } + pan_emit_bifrost_tiler_heap(dev, t.cpu); mali_ptr heap = t.gpu; t = panfrost_pool_alloc_desc(&batch->pool, BIFROST_TILER); - pan_pack(t.cpu, BIFROST_TILER, tiler) { - tiler.hierarchy_mask = 0x28; - tiler.fb_width = batch->key.width; - tiler.fb_height = batch->key.height; - tiler.heap = heap; + pan_emit_bifrost_tiler(dev, batch->key.width, batch->key.height, + util_framebuffer_get_num_samples(&batch->key), + heap, t.cpu); - /* Must match framebuffer descriptor */ - unsigned samples = util_framebuffer_get_num_samples(&batch->key); - tiler.sample_pattern = panfrost_sample_pattern(samples); - } - - batch->tiler_meta = t.gpu; - return batch->tiler_meta; + batch->tiler_ctx.bifrost = t.gpu; + return batch->tiler_ctx.bifrost; } -struct panfrost_bo * -panfrost_batch_get_tiler_dummy(struct panfrost_batch *batch) +static void +panfrost_batch_to_fb_info(const struct panfrost_batch *batch, + struct pan_fb_info *fb, + struct pan_image_view *rts, + struct pan_image_view *zs, + struct pan_image_view *s, + bool reserve) { - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); + memset(fb, 0, sizeof(*fb)); + memset(rts, 0, sizeof(*rts) * 8); + memset(zs, 0, sizeof(*zs)); + memset(s, 0, sizeof(*s)); - uint32_t create_flags = 0; + fb->width = batch->key.width; + fb->height = batch->key.height; + fb->extent.minx = batch->minx; + fb->extent.miny = batch->miny; + fb->extent.maxx = batch->maxx - 1; + fb->extent.maxy = batch->maxy - 1; + fb->nr_samples = util_framebuffer_get_num_samples(&batch->key); + fb->rt_count = batch->key.nr_cbufs; - if (batch->tiler_dummy) - return batch->tiler_dummy; + static const unsigned char id_swz[] = { + PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W, + }; - if (!(dev->quirks & MIDGARD_NO_HIER_TILING)) - create_flags = PAN_BO_INVISIBLE; + for (unsigned i = 0; i < fb->rt_count; i++) { + struct pipe_surface *surf = batch->key.cbufs[i]; - batch->tiler_dummy = panfrost_batch_create_bo(batch, 4096, - create_flags, - PAN_BO_ACCESS_PRIVATE | - PAN_BO_ACCESS_RW | - PAN_BO_ACCESS_VERTEX_TILER | - PAN_BO_ACCESS_FRAGMENT); - assert(batch->tiler_dummy); - return batch->tiler_dummy; + if (!surf) + continue; + + struct panfrost_resource *prsrc = pan_resource(surf->texture); + unsigned mask = PIPE_CLEAR_COLOR0 << i; + + if (batch->clear & mask) { + fb->rts[i].clear = true; + memcpy(fb->rts[i].clear_value, batch->clear_color[i], + sizeof((fb->rts[i].clear_value))); + } + + /* Discard RTs that have no draws or clear. */ + if (!reserve && !((batch->clear | batch->draws) & mask)) + fb->rts[i].discard = true; + + rts[i].format = surf->format; + rts[i].dim = MALI_TEXTURE_DIMENSION_2D; + rts[i].last_level = rts[i].first_level = surf->u.tex.level; + rts[i].first_layer = surf->u.tex.first_layer; + rts[i].last_layer = surf->u.tex.last_layer; + rts[i].image = &prsrc->image; + memcpy(rts[i].swizzle, id_swz, sizeof(rts[i].swizzle)); + fb->rts[i].state = &prsrc->state; + fb->rts[i].view = &rts[i]; + + /* Preload if the RT is read or updated */ + if (!(batch->clear & mask) && + ((batch->read & mask) || + ((batch->draws & mask) && + fb->rts[i].state->slices[fb->rts[i].view->first_level].data_valid))) + fb->rts[i].preload = true; + + } + + const struct pan_image_view *s_view = NULL, *z_view = NULL; + const struct pan_image_state *s_state = NULL, *z_state = NULL; + + if (batch->key.zsbuf) { + struct pipe_surface *surf = batch->key.zsbuf; + struct panfrost_resource *prsrc = pan_resource(surf->texture); + + zs->format = surf->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT ? + PIPE_FORMAT_Z32_FLOAT : surf->format; + zs->dim = MALI_TEXTURE_DIMENSION_2D; + zs->last_level = zs->first_level = surf->u.tex.level; + zs->first_layer = surf->u.tex.first_layer; + zs->last_layer = surf->u.tex.last_layer; + zs->image = &prsrc->image; + memcpy(zs->swizzle, id_swz, sizeof(zs->swizzle)); + fb->zs.view.zs = zs; + fb->zs.state.zs = &prsrc->state; + z_view = zs; + z_state = &prsrc->state; + if (util_format_is_depth_and_stencil(zs->format)) { + s_view = zs; + s_state = &prsrc->state; + } + + if (prsrc->separate_stencil) { + s->format = PIPE_FORMAT_S8_UINT; + s->dim = MALI_TEXTURE_DIMENSION_2D; + s->last_level = s->first_level = surf->u.tex.level; + s->first_layer = surf->u.tex.first_layer; + s->last_layer = surf->u.tex.last_layer; + s->image = &prsrc->separate_stencil->image; + memcpy(s->swizzle, id_swz, sizeof(s->swizzle)); + fb->zs.view.s = s; + fb->zs.state.s = &prsrc->separate_stencil->state; + s_view = s; + s_state = &prsrc->separate_stencil->state; + } + } + + if (batch->clear & PIPE_CLEAR_DEPTH) { + fb->zs.clear.z = true; + fb->zs.clear_value.depth = batch->clear_depth; + } + + if (batch->clear & PIPE_CLEAR_STENCIL) { + fb->zs.clear.s = true; + fb->zs.clear_value.stencil = batch->clear_stencil; + } + + /* Discard if Z/S are not updated */ + if (!reserve && !((batch->draws | batch->clear) & PIPE_CLEAR_DEPTH)) + fb->zs.discard.z = true; + + if (!reserve && !((batch->draws | batch->clear) & PIPE_CLEAR_STENCIL)) + fb->zs.discard.s = true; + + if (!fb->zs.clear.z && + ((batch->read & PIPE_CLEAR_DEPTH) || + ((batch->draws & PIPE_CLEAR_DEPTH) && + z_state->slices[z_view->first_level].data_valid))) + fb->zs.preload.z = true; + + if (!fb->zs.clear.s && + ((batch->read & PIPE_CLEAR_STENCIL) || + ((batch->draws & PIPE_CLEAR_STENCIL) && + s_state->slices[s_view->first_level].data_valid))) + fb->zs.preload.s = true; + + /* Preserve both component if we have a combined ZS view and + * one component needs to be preserved. + */ + if (s_view == z_view && fb->zs.discard.z != fb->zs.discard.s) { + bool valid = z_state->slices[z_view->first_level].data_valid; + + fb->zs.discard.z = false; + fb->zs.discard.s = false; + fb->zs.preload.z = !fb->zs.clear.z && valid; + fb->zs.preload.s = !fb->zs.clear.s && valid; + } } mali_ptr @@ -742,200 +878,69 @@ panfrost_batch_reserve_framebuffer(struct panfrost_batch *batch) { struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - /* If we haven't, reserve space for the thread storage descriptor (or a - * full framebuffer descriptor on Midgard) */ + if (batch->framebuffer.gpu) + return batch->framebuffer.gpu; - if (!batch->framebuffer.gpu) { - if (pan_is_bifrost(dev)) { - batch->framebuffer = - panfrost_pool_alloc_desc(&batch->pool, - LOCAL_STORAGE); - } else if (dev->quirks & MIDGARD_SFBD) { - batch->framebuffer = - panfrost_pool_alloc_desc(&batch->pool, - SINGLE_TARGET_FRAMEBUFFER); - } else { - batch->framebuffer = - panfrost_pool_alloc_desc(&batch->pool, - MULTI_TARGET_FRAMEBUFFER); - /* Tag the pointer */ - batch->framebuffer.gpu |= MALI_FBD_TAG_IS_MFBD; - } - } + /* If we haven't, reserve space for a framebuffer descriptor */ + + struct pan_image_view rts[8]; + struct pan_image_view zs; + struct pan_image_view s; + struct pan_fb_info fb; + + panfrost_batch_to_fb_info(batch, &fb, rts, &zs, &s, true); + + unsigned zs_crc_count = pan_fbd_has_zs_crc_ext(dev, &fb) ? 1 : 0; + unsigned rt_count = MAX2(fb.rt_count, 1); + batch->framebuffer = + (dev->quirks & MIDGARD_SFBD) ? + panfrost_pool_alloc_desc(&batch->pool, SINGLE_TARGET_FRAMEBUFFER) : + panfrost_pool_alloc_desc_aggregate(&batch->pool, + PAN_DESC(MULTI_TARGET_FRAMEBUFFER), + PAN_DESC_ARRAY(zs_crc_count, ZS_CRC_EXTENSION), + PAN_DESC_ARRAY(rt_count, RENDER_TARGET)); + + /* Add the MFBD tag now, other tags will be added when emitting the + * FB desc. + */ + if (!(dev->quirks & MIDGARD_SFBD)) + batch->framebuffer.gpu |= MALI_FBD_TAG_IS_MFBD; return batch->framebuffer.gpu; } -static void -panfrost_load_surface(struct panfrost_batch *batch, struct pipe_surface *surf, unsigned loc) +mali_ptr +panfrost_batch_reserve_tls(struct panfrost_batch *batch) { - if (!surf) - return; + struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - struct panfrost_resource *rsrc = pan_resource(surf->texture); - unsigned level = surf->u.tex.level; + /* If we haven't, reserve space for the thread storage descriptor */ - if (!rsrc->state.slices[level].data_valid) - return; + if (batch->tls.gpu) + return batch->tls.gpu; - /* Clamp the rendering area to the damage extent. The - * KHR_partial_update() spec states that trying to render outside of - * the damage region is "undefined behavior", so we should be safe. - */ - unsigned damage_width = (rsrc->damage.extent.maxx - rsrc->damage.extent.minx); - unsigned damage_height = (rsrc->damage.extent.maxy - rsrc->damage.extent.miny); - - if (damage_width && damage_height) { - panfrost_batch_intersection_scissor(batch, - rsrc->damage.extent.minx, - rsrc->damage.extent.miny, - rsrc->damage.extent.maxx, - rsrc->damage.extent.maxy); - } - - /* Note: this may not equal surf->texture->format, we reinterpret */ - enum pipe_format format = surf->format; - - if (loc == FRAG_RESULT_DEPTH) { - if (!util_format_has_depth(util_format_description(format))) - return; - - format = util_format_get_depth_only(format); - } else if (loc == FRAG_RESULT_STENCIL) { - if (!util_format_has_stencil(util_format_description(format))) - return; - - if (rsrc->separate_stencil) { - rsrc = rsrc->separate_stencil; - format = rsrc->base.format; - } - - format = util_format_stencil_only(format); - } - - struct pan_image_view iview = { - .format = format, - .dim = MALI_TEXTURE_DIMENSION_2D, - .first_level = level, - .last_level = level, - .first_layer = surf->u.tex.first_layer, - .last_layer = surf->u.tex.last_layer, - .swizzle = { - PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, - PIPE_SWIZZLE_Z, PIPE_SWIZZLE_W, - }, - .image = &rsrc->image, - }; - - mali_ptr blend_shader = 0; - - if (loc >= FRAG_RESULT_DATA0 && - !panfrost_blend_format(format).internal) { - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - struct panfrost_bo *bo = - panfrost_batch_create_bo(batch, 4096, - PAN_BO_EXECUTE, - PAN_BO_ACCESS_PRIVATE | - PAN_BO_ACCESS_READ | - PAN_BO_ACCESS_FRAGMENT); - unsigned rt = loc - FRAG_RESULT_DATA0; - struct pan_blend_state blend_state = { - .rt_count = rt + 1, - }; - - blend_state.rts[rt] = (struct pan_blend_rt_state) { - .format = format, - .nr_samples = rsrc->base.nr_samples, - .equation = { - .blend_enable = true, - .rgb_src_factor = BLEND_FACTOR_ZERO, - .rgb_invert_src_factor = true, - .rgb_dst_factor = BLEND_FACTOR_ZERO, - .rgb_func = BLEND_FUNC_ADD, - .alpha_src_factor = BLEND_FACTOR_ZERO, - .alpha_invert_src_factor = true, - .alpha_dst_factor = BLEND_FACTOR_ZERO, - .alpha_func = BLEND_FUNC_ADD, - .color_mask = 0xf, - }, - }; - - pthread_mutex_lock(&dev->blend_shaders.lock); - struct pan_blend_shader_variant *b = - pan_blend_get_shader_locked(dev, &blend_state, - loc - FRAG_RESULT_DATA0); - - assert(pan_is_bifrost(dev) || b->work_reg_count <= 4); - memcpy(bo->ptr.cpu, b->binary.data, b->binary.size); - - blend_shader = bo->ptr.gpu | b->first_tag; - pthread_mutex_unlock(&dev->blend_shaders.lock); - } - - float rect[] = { - 0.0, rsrc->base.height0, 0.0, 1.0, - rsrc->base.width0, rsrc->base.height0, 0.0, 1.0, - 0.0, 0.0, 0.0, 1.0, - - rsrc->base.width0, rsrc->base.height0, 0.0, 1.0, - 0.0, 0.0, 0.0, 1.0, - rsrc->base.width0, 0.0, 0.0, 1.0, - }; - - mali_ptr vertices = - panfrost_pool_upload_aligned(&batch->pool, rect, sizeof(rect), 64); - - if (pan_is_bifrost(batch->pool.dev)) { - mali_ptr tiler = - panfrost_batch_get_bifrost_tiler(batch, 6); - panfrost_load_bifrost(&batch->pool, &batch->scoreboard, - blend_shader, - batch->framebuffer.gpu, - tiler, - vertices, 6, - &iview, loc); + if (pan_is_bifrost(dev)) { + batch->tls = panfrost_pool_alloc_desc(&batch->pool, LOCAL_STORAGE); } else { - panfrost_load_midg(&batch->pool, &batch->scoreboard, - blend_shader, - batch->framebuffer.gpu, - vertices, 6, - &iview, loc); + /* On Midgard, the FB descriptor contains a thread storage + * descriptor, and tiler jobs need more than thread storage + * info. Let's point to the FB desc in that case. + */ + panfrost_batch_reserve_framebuffer(batch); + batch->tls = batch->framebuffer; } - panfrost_batch_add_bo(batch, batch->pool.dev->blit_shaders.bo, - PAN_BO_ACCESS_SHARED | PAN_BO_ACCESS_READ | PAN_BO_ACCESS_FRAGMENT); + return batch->tls.gpu; } static void -panfrost_batch_draw_wallpaper(struct panfrost_batch *batch) +panfrost_batch_draw_wallpaper(struct panfrost_batch *batch, + struct pan_fb_info *fb) { - panfrost_batch_reserve_framebuffer(batch); + struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - /* Assume combined. If either depth or stencil is written, they will - * both be written so we need to be careful for reloading */ - - unsigned reload = batch->draws | batch->read; - - if (reload & PIPE_CLEAR_DEPTHSTENCIL) - reload |= PIPE_CLEAR_DEPTHSTENCIL; - - /* Mask of buffers which need reload since they are not cleared and - * they are drawn. (If they are cleared, reload is useless; if they are - * not drawn or read and also not cleared, we can generally omit the - * attachment at the framebuffer descriptor level */ - - reload &= ~batch->clear; - - for (unsigned i = 0; i < batch->key.nr_cbufs; ++i) { - if (reload & (PIPE_CLEAR_COLOR0 << i)) - panfrost_load_surface(batch, batch->key.cbufs[i], FRAG_RESULT_DATA0 + i); - } - - if (reload & PIPE_CLEAR_DEPTH) - panfrost_load_surface(batch, batch->key.zsbuf, FRAG_RESULT_DEPTH); - - if (reload & PIPE_CLEAR_STENCIL) - panfrost_load_surface(batch, batch->key.zsbuf, FRAG_RESULT_STENCIL); + pan_preload_fb(&batch->pool, &batch->scoreboard, fb, batch->tls.gpu, + pan_is_bifrost(dev) ? batch->tiler_ctx.bifrost : 0); } static void @@ -1044,7 +1049,9 @@ panfrost_batch_submit_ioctl(struct panfrost_batch *batch, * implicit dep between them) */ static int -panfrost_batch_submit_jobs(struct panfrost_batch *batch, uint32_t in_sync, uint32_t out_sync) +panfrost_batch_submit_jobs(struct panfrost_batch *batch, + const struct pan_fb_info *fb, + uint32_t in_sync, uint32_t out_sync) { struct panfrost_device *dev = pan_device(batch->ctx->base.screen); bool has_draws = batch->scoreboard.first_job; @@ -1073,7 +1080,7 @@ panfrost_batch_submit_jobs(struct panfrost_batch *batch, uint32_t in_sync, uint3 * *only* clears, since otherwise the tiler structures will be * uninitialized leading to faults (or state leaks) */ - mali_ptr fragjob = panfrost_fragment_job(batch, has_tiler); + mali_ptr fragjob = panfrost_emit_fragment_job(batch, fb); ret = panfrost_batch_submit_ioctl(batch, fragjob, PANFROST_JD_REQ_FS, 0, out_sync); @@ -1107,28 +1114,33 @@ panfrost_batch_submit(struct panfrost_batch *batch, if (!batch->scoreboard.first_job && !batch->clear) goto out; - panfrost_batch_draw_wallpaper(batch); + if (batch->scoreboard.first_tiler || batch->clear) + panfrost_batch_reserve_framebuffer(batch); + + struct pan_fb_info fb; + struct pan_image_view rts[8], zs, s; + + panfrost_batch_to_fb_info(batch, &fb, rts, &zs, &s, false); + + panfrost_batch_reserve_tls(batch); + panfrost_batch_draw_wallpaper(batch, &fb); + + + if (!pan_is_bifrost(dev)) { + mali_ptr polygon_list = panfrost_batch_get_polygon_list(batch); + + panfrost_scoreboard_initialize_tiler(&batch->pool, &batch->scoreboard, polygon_list); + } /* Now that all draws are in, we can finally prepare the * FBD for the batch */ - if (batch->framebuffer.gpu && batch->scoreboard.first_job) { - struct panfrost_context *ctx = batch->ctx; - struct pipe_context *gallium = (struct pipe_context *) ctx; - struct panfrost_device *dev = pan_device(gallium->screen); + panfrost_emit_tls(batch); - if (dev->quirks & MIDGARD_SFBD) - panfrost_attach_sfbd(batch, ~0); - else - panfrost_attach_mfbd(batch, ~0); - } + if (batch->framebuffer.gpu) + panfrost_emit_fbd(batch, &fb); - mali_ptr polygon_list = panfrost_batch_get_polygon_list(batch, - MALI_MIDGARD_TILER_MINIMUM_HEADER_SIZE); - - panfrost_scoreboard_initialize_tiler(&batch->pool, &batch->scoreboard, polygon_list); - - ret = panfrost_batch_submit_jobs(batch, in_sync, out_sync); + ret = panfrost_batch_submit_jobs(batch, &fb, in_sync, out_sync); if (ret && dev->debug & PAN_DBG_MSGS) fprintf(stderr, "panfrost_batch_submit failed: %d\n", ret); diff --git a/src/gallium/drivers/panfrost/pan_job.h b/src/gallium/drivers/panfrost/pan_job.h index 088a3af2d64..080c7b9a60e 100644 --- a/src/gallium/drivers/panfrost/pan_job.h +++ b/src/gallium/drivers/panfrost/pan_job.h @@ -28,6 +28,7 @@ #include "util/u_dynarray.h" #include "pipe/p_state.h" +#include "pan_cs.h" #include "pan_pool.h" #include "pan_resource.h" #include "pan_scoreboard.h" @@ -108,17 +109,14 @@ struct panfrost_batch { /* Shared memory BO bound to the batch, or NULL if none bound yet */ struct panfrost_bo *shared_memory; - /* Tiler heap BO bound to the batch, or NULL if none bound yet */ - struct panfrost_bo *tiler_heap; - - /* Dummy tiler BO bound to the batch, or NULL if none bound yet */ - struct panfrost_bo *tiler_dummy; - /* Framebuffer descriptor. */ struct panfrost_ptr framebuffer; - /* Bifrost tiler meta descriptor. */ - mali_ptr tiler_meta; + /* Thread local storage descriptor. */ + struct panfrost_ptr tls; + + /* Tiler context */ + struct pan_tiler_context tiler_ctx; /* Output sync object. Only valid when submitted is true. */ struct panfrost_batch_fence *out_sync; @@ -179,12 +177,6 @@ panfrost_batch_get_scratchpad(struct panfrost_batch *batch, unsigned size, unsig struct panfrost_bo * panfrost_batch_get_shared_memory(struct panfrost_batch *batch, unsigned size, unsigned workgroup_count); -mali_ptr -panfrost_batch_get_polygon_list(struct panfrost_batch *batch, unsigned size); - -struct panfrost_bo * -panfrost_batch_get_tiler_dummy(struct panfrost_batch *batch); - void panfrost_batch_clear(struct panfrost_batch *batch, unsigned buffers, @@ -207,4 +199,7 @@ panfrost_batch_get_bifrost_tiler(struct panfrost_batch *batch, unsigned vertex_c mali_ptr panfrost_batch_reserve_framebuffer(struct panfrost_batch *batch); +mali_ptr +panfrost_batch_reserve_tls(struct panfrost_batch *batch); + #endif diff --git a/src/gallium/drivers/panfrost/pan_mfbd.c b/src/gallium/drivers/panfrost/pan_mfbd.c deleted file mode 100644 index b3f6fc8596b..00000000000 --- a/src/gallium/drivers/panfrost/pan_mfbd.c +++ /dev/null @@ -1,663 +0,0 @@ -/* - * Copyright (C) 2019-2020 Collabora, Ltd. - * Copyright 2018-2019 Alyssa Rosenzweig - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include "pan_bo.h" -#include "pan_context.h" -#include "pan_cmdstream.h" -#include "pan_util.h" -#include "panfrost-quirks.h" - - -static bool -panfrost_mfbd_has_zs_crc_ext(struct panfrost_batch *batch) -{ - if (batch->key.nr_cbufs == 1) { - struct pipe_surface *surf = batch->key.cbufs[0]; - - if (surf->texture && - pan_resource(surf->texture)->image.layout.crc_mode != PAN_IMAGE_CRC_NONE) - return true; - } - - if (batch->key.zsbuf && - ((batch->clear | batch->draws) & PIPE_CLEAR_DEPTHSTENCIL)) - return true; - - return false; -} - -static enum mali_mfbd_color_format -panfrost_mfbd_raw_format(unsigned bits) -{ - switch (bits) { - case 8: return MALI_MFBD_COLOR_FORMAT_RAW8; - case 16: return MALI_MFBD_COLOR_FORMAT_RAW16; - case 24: return MALI_MFBD_COLOR_FORMAT_RAW24; - case 32: return MALI_MFBD_COLOR_FORMAT_RAW32; - case 48: return MALI_MFBD_COLOR_FORMAT_RAW48; - case 64: return MALI_MFBD_COLOR_FORMAT_RAW64; - case 96: return MALI_MFBD_COLOR_FORMAT_RAW96; - case 128: return MALI_MFBD_COLOR_FORMAT_RAW128; - case 192: return MALI_MFBD_COLOR_FORMAT_RAW192; - case 256: return MALI_MFBD_COLOR_FORMAT_RAW256; - case 384: return MALI_MFBD_COLOR_FORMAT_RAW384; - case 512: return MALI_MFBD_COLOR_FORMAT_RAW512; - case 768: return MALI_MFBD_COLOR_FORMAT_RAW768; - case 1024: return MALI_MFBD_COLOR_FORMAT_RAW1024; - case 1536: return MALI_MFBD_COLOR_FORMAT_RAW1536; - case 2048: return MALI_MFBD_COLOR_FORMAT_RAW2048; - default: unreachable("invalid raw bpp"); - } -} - -static void -panfrost_mfbd_rt_init_format(const struct panfrost_device *dev, - struct pipe_surface *surf, - struct MALI_RENDER_TARGET *rt) -{ - struct panfrost_resource *rsrc = pan_resource(surf->texture); - enum pipe_format format = - drm_is_afbc(rsrc->image.layout.modifier) ? - panfrost_afbc_format_fixup(dev, surf->format) : - surf->format; - - /* Explode details on the format */ - - const struct util_format_description *desc = - util_format_description(format); - - /* The swizzle for rendering is inverted from texturing */ - - unsigned char swizzle[4]; - panfrost_invert_swizzle(desc->swizzle, swizzle); - - rt->swizzle = panfrost_translate_swizzle_4(swizzle); - - /* Fill in accordingly, defaulting to 8-bit UNORM */ - - if (desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) - rt->srgb = true; - - struct pan_blendable_format fmt = panfrost_blend_format(surf->format); - - if (fmt.internal) { - rt->internal_format = fmt.internal; - rt->writeback_format = fmt.writeback; - } else { - /* Construct RAW internal/writeback, where internal is - * specified logarithmically (round to next power-of-two). - * Offset specified from RAW8, where 8 = 2^3 */ - - unsigned bits = desc->block.bits; - unsigned offset = util_logbase2_ceil(bits) - 3; - assert(offset <= 4); - - rt->internal_format = - MALI_COLOR_BUFFER_INTERNAL_FORMAT_RAW8 + offset; - - rt->writeback_format = panfrost_mfbd_raw_format(bits); - } -} - -static void -panfrost_mfbd_rt_set_buf(struct pipe_surface *surf, - struct MALI_RENDER_TARGET *rt) -{ - struct panfrost_device *dev = pan_device(surf->context->screen); - struct panfrost_resource *rsrc = pan_resource(surf->texture); - unsigned level = surf->u.tex.level; - unsigned first_layer = surf->u.tex.first_layer; - assert(surf->u.tex.last_layer == first_layer); - int row_stride = rsrc->image.layout.slices[level].row_stride; - - /* Only set layer_stride for layered MSAA rendering */ - - unsigned nr_samples = surf->texture->nr_samples; - unsigned layer_stride = (nr_samples > 1) ? rsrc->image.layout.slices[level].surface_stride : 0; - - if (layer_stride) - rt->writeback_msaa = MALI_MSAA_LAYERED; - else if (surf->nr_samples) - rt->writeback_msaa = MALI_MSAA_AVERAGE; - else - rt->writeback_msaa = MALI_MSAA_SINGLE; - - panfrost_mfbd_rt_init_format(dev, surf, rt); - - if (rsrc->image.layout.modifier == DRM_FORMAT_MOD_LINEAR) { - mali_ptr base = panfrost_get_texture_address(rsrc, level, first_layer, 0); - - if (dev->arch >= 7) - rt->bifrost_v7.writeback_block_format = MALI_BLOCK_FORMAT_V7_LINEAR; - else - rt->midgard.writeback_block_format = MALI_BLOCK_FORMAT_LINEAR; - - rt->rgb.base = base; - rt->rgb.row_stride = row_stride; - rt->rgb.surface_stride = layer_stride; - } else if (rsrc->image.layout.modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { - mali_ptr base = panfrost_get_texture_address(rsrc, level, first_layer, 0); - - if (dev->arch >= 7) - rt->bifrost_v7.writeback_block_format = MALI_BLOCK_FORMAT_V7_TILED_U_INTERLEAVED; - else - rt->midgard.writeback_block_format = MALI_BLOCK_FORMAT_TILED_U_INTERLEAVED; - - rt->rgb.base = base; - rt->rgb.row_stride = row_stride; - rt->rgb.surface_stride = layer_stride; - } else if (drm_is_afbc(rsrc->image.layout.modifier)) { - const struct pan_image_slice_layout *slice = &rsrc->image.layout.slices[level]; - - if (dev->arch >= 7) - rt->bifrost_v7.writeback_block_format = MALI_BLOCK_FORMAT_V7_AFBC; - else - rt->midgard.writeback_block_format = MALI_BLOCK_FORMAT_AFBC; - - if (pan_is_bifrost(dev)) { - rt->afbc.row_stride = slice->afbc.row_stride / - AFBC_HEADER_BYTES_PER_TILE; - rt->bifrost_afbc.afbc_wide_block_enable = - panfrost_block_dim(rsrc->image.layout.modifier, true, 0) > 16; - } else { - rt->afbc.chunk_size = 9; - rt->midgard_afbc.sparse = true; - rt->afbc.body_size = slice->afbc.body_size; - assert(rt->afbc.body_size > 0); - } - - panfrost_get_afbc_pointers(rsrc, level, first_layer, - &rt->afbc.header, - &rt->afbc.body); - - if (rsrc->image.layout.modifier & AFBC_FORMAT_MOD_YTR) - rt->afbc.yuv_transform_enable = true; - } else { - unreachable("Invalid mod"); - } -} - -static void -panfrost_mfbd_emit_rt(struct panfrost_batch *batch, - void *rtp, struct pipe_surface *surf, - unsigned rt_offset, unsigned rt_idx) -{ - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - - pan_pack(rtp, RENDER_TARGET, rt) { - rt.clean_pixel_write_enable = true; - if (surf) { - rt.write_enable = true; - rt.dithering_enable = true; - rt.internal_buffer_offset = rt_offset; - panfrost_mfbd_rt_set_buf(surf, &rt); - } else { - rt.internal_format = MALI_COLOR_BUFFER_INTERNAL_FORMAT_R8G8B8A8; - rt.internal_buffer_offset = rt_offset; - if (dev->arch >= 7) { - rt.bifrost_v7.writeback_block_format = MALI_BLOCK_FORMAT_V7_TILED_U_INTERLEAVED; - rt.dithering_enable = true; - } - } - - if (batch->clear & (PIPE_CLEAR_COLOR0 << rt_idx)) { - rt.clear.color_0 = batch->clear_color[rt_idx][0]; - rt.clear.color_1 = batch->clear_color[rt_idx][1]; - rt.clear.color_2 = batch->clear_color[rt_idx][2]; - rt.clear.color_3 = batch->clear_color[rt_idx][3]; - } - } -} - -static enum mali_z_internal_format -get_z_internal_format(struct panfrost_batch *batch) -{ - struct pipe_surface *zs_surf = batch->key.zsbuf; - - /* Default to 24 bit depth if there's no surface. */ - if (!zs_surf || !((batch->clear | batch->draws) & PIPE_CLEAR_DEPTHSTENCIL)) - return MALI_Z_INTERNAL_FORMAT_D24; - - return panfrost_get_z_internal_format(zs_surf->format); -} - -static void -panfrost_mfbd_zs_crc_ext_set_bufs(struct panfrost_batch *batch, - struct MALI_ZS_CRC_EXTENSION *ext, - struct pan_image_slice_state **checksum_slice) -{ - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - - /* Checksumming only works with a single render target */ - if (batch->key.nr_cbufs == 1 && batch->key.cbufs[0] && - ((batch->clear | batch->draws) & PIPE_CLEAR_COLOR0)) { - struct pipe_surface *c_surf = batch->key.cbufs[0]; - struct panfrost_resource *rsrc = pan_resource(c_surf->texture); - - if (rsrc->image.layout.crc_mode != PAN_IMAGE_CRC_NONE) { - unsigned level = c_surf->u.tex.level; - struct pan_image_slice_layout *slice = &rsrc->image.layout.slices[level]; - - *checksum_slice = &rsrc->state.slices[level]; - - ext->crc_row_stride = slice->crc.stride; - if (rsrc->image.layout.crc_mode == PAN_IMAGE_CRC_OOB) { - ext->crc_base = rsrc->image.crc.bo->ptr.gpu + - slice->crc.offset; - } else { - ext->crc_base = rsrc->image.data.bo->ptr.gpu + - slice->crc.offset; - } - - if ((batch->clear & PIPE_CLEAR_COLOR0) && dev->arch >= 7) { - ext->crc_clear_color = batch->clear_color[0][0] | - 0xc000000000000000 | - ((uint64_t)batch->clear_color[0][0] & 0xffff) << 32; - } - } - } - - struct pipe_surface *zs_surf = batch->key.zsbuf; - - if (!((batch->clear | batch->draws) & PIPE_CLEAR_DEPTHSTENCIL)) - zs_surf = NULL; - - if (!zs_surf) - return; - - struct panfrost_resource *rsrc = pan_resource(zs_surf->texture); - unsigned nr_samples = MAX2(zs_surf->texture->nr_samples, 1); - unsigned level = zs_surf->u.tex.level; - unsigned first_layer = zs_surf->u.tex.first_layer; - assert(zs_surf->u.tex.last_layer == first_layer); - - if (dev->arch < 7) - ext->zs_msaa = nr_samples > 1 ? MALI_MSAA_LAYERED : MALI_MSAA_SINGLE; - else - ext->zs_msaa_v7 = nr_samples > 1 ? MALI_MSAA_LAYERED : MALI_MSAA_SINGLE; - - if (drm_is_afbc(rsrc->image.layout.modifier)) { - struct pan_image_slice_layout *slice = &rsrc->image.layout.slices[level]; - - panfrost_get_afbc_pointers(rsrc, level, first_layer, - &ext->zs_afbc_header, - &ext->zs_afbc_body); - - if (dev->arch >= 7) - ext->zs_block_format_v7 = MALI_BLOCK_FORMAT_V7_AFBC; - else - ext->zs_block_format = MALI_BLOCK_FORMAT_AFBC; - - if (pan_is_bifrost(dev)) { - ext->zs_afbc_row_stride = slice->afbc.row_stride / - AFBC_HEADER_BYTES_PER_TILE; - } else { - ext->zs_block_format = MALI_BLOCK_FORMAT_AFBC; - ext->zs_afbc_body_size = 0x1000; - ext->zs_afbc_chunk_size = 9; - ext->zs_afbc_sparse = true; - } - } else { - assert(rsrc->image.layout.modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED || - rsrc->image.layout.modifier == DRM_FORMAT_MOD_LINEAR); - mali_ptr base = panfrost_get_texture_address(rsrc, level, first_layer, 0); - - /* TODO: Z32F(S8) support, which is always linear */ - - ext->zs_writeback_base = base; - ext->zs_writeback_row_stride = - rsrc->image.layout.slices[level].row_stride; - ext->zs_writeback_surface_stride = - (nr_samples > 1) ? - rsrc->image.layout.slices[level].surface_stride : 0; - - if (rsrc->image.layout.modifier == DRM_FORMAT_MOD_LINEAR) { - if (dev->arch >= 7) - ext->zs_block_format_v7 = MALI_BLOCK_FORMAT_V7_LINEAR; - else - ext->zs_block_format = MALI_BLOCK_FORMAT_LINEAR; - } else { - if (dev->arch >= 7) - ext->zs_block_format_v7 = MALI_BLOCK_FORMAT_V7_TILED_U_INTERLEAVED; - else - ext->zs_block_format = MALI_BLOCK_FORMAT_TILED_U_INTERLEAVED; - } - } - - switch (zs_surf->format) { - case PIPE_FORMAT_Z16_UNORM: - ext->zs_write_format = MALI_ZS_FORMAT_D16; - break; - case PIPE_FORMAT_Z24_UNORM_S8_UINT: - ext->zs_write_format = MALI_ZS_FORMAT_D24S8; - ext->s_writeback_base = ext->zs_writeback_base; - break; - case PIPE_FORMAT_Z24X8_UNORM: - ext->zs_write_format = MALI_ZS_FORMAT_D24X8; - break; - case PIPE_FORMAT_Z32_FLOAT: - ext->zs_write_format = MALI_ZS_FORMAT_D32; - break; - case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: - /* Midgard/Bifrost support interleaved depth/stencil - * buffers, but we always treat them as multi-planar. - */ - ext->zs_write_format = MALI_ZS_FORMAT_D32; - ext->s_write_format = MALI_S_FORMAT_S8; - if (dev->arch < 7) { - ext->s_block_format = ext->zs_block_format; - ext->s_msaa = ext->zs_msaa; - } else { - ext->s_block_format_v7 = ext->zs_block_format_v7; - ext->s_msaa_v7 = ext->zs_msaa_v7; - } - - struct panfrost_resource *stencil = rsrc->separate_stencil; - struct pan_image_slice_layout *stencil_slice = - &stencil->image.layout.slices[level]; - - ext->s_writeback_base = - panfrost_get_texture_address(stencil, level, first_layer, 0); - ext->s_writeback_row_stride = stencil_slice->row_stride; - ext->s_writeback_surface_stride = - (nr_samples > 1) ? stencil_slice->surface_stride : 0; - break; - default: - unreachable("Unsupported depth/stencil format."); - } -} - -static void -panfrost_mfbd_emit_zs_crc_ext(struct panfrost_batch *batch, void *extp, - struct pan_image_slice_state **checksum_slice) -{ - pan_pack(extp, ZS_CRC_EXTENSION, ext) { - ext.zs_clean_pixel_write_enable = true; - panfrost_mfbd_zs_crc_ext_set_bufs(batch, &ext, checksum_slice); - } -} - -/* Measure format as it appears in the tile buffer */ - -static unsigned -pan_bytes_per_pixel_tib(enum pipe_format format) -{ - if (panfrost_blend_format(format).internal) { - /* Blendable formats are always 32-bits in the tile buffer, - * extra bits are used as padding or to dither */ - return 4; - } else { - /* Non-blendable formats are raw, rounded up to the nearest - * power-of-two size */ - unsigned bytes = util_format_get_blocksize(format); - return util_next_power_of_two(bytes); - } -} - -/* Calculates the internal color buffer size and tile size based on the number - * of RT, the format and the number of pixels. If things do not fit in 4KB, we - * shrink the tile size to make it fit. - */ - -static unsigned -pan_internal_cbuf_size(struct panfrost_batch *batch, unsigned *tile_size) -{ - unsigned total_size = 0; - - *tile_size = 16 * 16; - for (int cb = 0; cb < batch->key.nr_cbufs; ++cb) { - struct pipe_surface *surf = batch->key.cbufs[cb]; - - if (!((batch->clear | batch->draws) & (PIPE_CLEAR_COLOR0 << cb))) - surf = NULL; - - if (!surf) - continue; - - unsigned nr_samples = MAX3(surf->nr_samples, surf->texture->nr_samples, 1); - total_size += pan_bytes_per_pixel_tib(surf->format) * - nr_samples * (*tile_size); - } - - /* We have a 4KB budget, let's reduce the tile size until it fits. */ - while (total_size > 4096) { - total_size >>= 1; - *tile_size >>= 1; - } - - /* Align on 1k. */ - total_size = ALIGN_POT(total_size, 1024); - - /* Minimum tile size is 4x4. */ - assert(*tile_size >= 4 * 4); - return total_size; -} - -static void -panfrost_mfbd_emit_local_storage(struct panfrost_batch *batch, void *fb) -{ - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - - pan_section_pack(fb, MULTI_TARGET_FRAMEBUFFER, LOCAL_STORAGE, ls) { - if (batch->stack_size) { - unsigned shift = - panfrost_get_stack_shift(batch->stack_size); - struct panfrost_bo *bo = - panfrost_batch_get_scratchpad(batch, - batch->stack_size, - dev->thread_tls_alloc, - dev->core_count); - ls.tls_size = shift; - ls.tls_base_pointer = bo->ptr.gpu; - } - - ls.wls_instances = MALI_LOCAL_STORAGE_NO_WORKGROUP_MEM; - } -} - -static void -panfrost_mfbd_emit_midgard_tiler(struct panfrost_batch *batch, void *fb, - unsigned vertex_count) -{ - void *t = pan_section_ptr(fb, MULTI_TARGET_FRAMEBUFFER, TILER); - - panfrost_emit_midg_tiler(batch, t, vertex_count); - - /* All weights set to 0, nothing to do here */ - pan_section_pack(fb, MULTI_TARGET_FRAMEBUFFER, TILER_WEIGHTS, w); -} - -static void -panfrost_mfbd_emit_bifrost_parameters(struct panfrost_batch *batch, void *fb) -{ - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - - pan_section_pack(fb, MULTI_TARGET_FRAMEBUFFER, BIFROST_PARAMETERS, params) { - unsigned samples = util_framebuffer_get_num_samples(&batch->key); - params.sample_locations = panfrost_sample_positions(dev, panfrost_sample_pattern(samples)); - } -} - -static void -panfrost_mfbd_emit_bifrost_tiler(struct panfrost_batch *batch, void *fb, - unsigned vertex_count) -{ - pan_section_pack(fb, MULTI_TARGET_FRAMEBUFFER, BIFROST_TILER_POINTER, tiler) { - tiler.address = panfrost_batch_get_bifrost_tiler(batch, vertex_count); - } - pan_section_pack(fb, MULTI_TARGET_FRAMEBUFFER, BIFROST_PADDING, padding); -} - -void -panfrost_attach_mfbd(struct panfrost_batch *batch, unsigned vertex_count) -{ - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - void *fb = batch->framebuffer.cpu; - - panfrost_mfbd_emit_local_storage(batch, fb); - - if (pan_is_bifrost(dev)) - return; - - pan_section_pack(fb, MULTI_TARGET_FRAMEBUFFER, PARAMETERS, params) { - params.width = batch->key.width; - params.height = batch->key.height; - params.bound_max_x = batch->key.width - 1; - params.bound_max_y = batch->key.height - 1; - params.color_buffer_allocation = - pan_internal_cbuf_size(batch, ¶ms.effective_tile_size); - params.tie_break_rule = MALI_TIE_BREAK_RULE_MINUS_180_IN_0_OUT; - params.render_target_count = MAX2(batch->key.nr_cbufs, 1); - params.sample_count = util_framebuffer_get_num_samples(&batch->key); - params.sample_pattern = panfrost_sample_pattern(params.sample_count); - } - - panfrost_mfbd_emit_midgard_tiler(batch, fb, vertex_count); -} - -/* Creates an MFBD for the FRAGMENT section of the bound framebuffer */ - -mali_ptr -panfrost_mfbd_fragment(struct panfrost_batch *batch, bool has_draws) -{ - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - unsigned vertex_count = has_draws; - unsigned zs_crc_count = panfrost_mfbd_has_zs_crc_ext(batch) ? 1 : 0; - unsigned rt_count = MAX2(batch->key.nr_cbufs, 1); - - struct panfrost_ptr t = - panfrost_pool_alloc_desc_aggregate(&batch->pool, - PAN_DESC(MULTI_TARGET_FRAMEBUFFER), - PAN_DESC_ARRAY(zs_crc_count, ZS_CRC_EXTENSION), - PAN_DESC_ARRAY(rt_count, RENDER_TARGET)); - void *fb = t.cpu, *zs_crc_ext, *rts; - - if (panfrost_mfbd_has_zs_crc_ext(batch)) { - zs_crc_ext = fb + MALI_MULTI_TARGET_FRAMEBUFFER_LENGTH; - rts = zs_crc_ext + MALI_ZS_CRC_EXTENSION_LENGTH; - } else { - zs_crc_ext = NULL; - rts = fb + MALI_MULTI_TARGET_FRAMEBUFFER_LENGTH; - } - - struct pan_image_slice_state *checksum_slice = NULL; - - if (zs_crc_ext) - panfrost_mfbd_emit_zs_crc_ext(batch, zs_crc_ext, &checksum_slice); - - /* We always upload at least one dummy GL_NONE render target */ - - unsigned rt_descriptors = MAX2(batch->key.nr_cbufs, 1); - - /* Upload either the render target or a dummy GL_NONE target */ - - unsigned rt_offset = 0, tib_size; - unsigned internal_cbuf_size = pan_internal_cbuf_size(batch, &tib_size); - - for (int cb = 0; cb < rt_descriptors; ++cb) { - struct pipe_surface *surf = batch->key.cbufs[cb]; - void *rt = rts + (cb * MALI_RENDER_TARGET_LENGTH); - - if (!((batch->clear | batch->draws) & (PIPE_CLEAR_COLOR0 << cb))) - surf = NULL; - - panfrost_mfbd_emit_rt(batch, rt, surf, rt_offset, cb); - - if (surf) { - unsigned samples = MAX2(surf->nr_samples, surf->texture->nr_samples); - - rt_offset += pan_bytes_per_pixel_tib(surf->format) * tib_size * - MAX2(samples, 1); - - struct panfrost_resource *prsrc = pan_resource(surf->texture); - if (checksum_slice != &prsrc->state.slices[surf->u.tex.level]) - prsrc->state.slices[surf->u.tex.level].crc_valid = false; - } - } - - if (pan_is_bifrost(dev)) - panfrost_mfbd_emit_bifrost_parameters(batch, fb); - else - panfrost_mfbd_emit_local_storage(batch, fb); - - pan_section_pack(fb, MULTI_TARGET_FRAMEBUFFER, PARAMETERS, params) { - params.width = batch->key.width; - params.height = batch->key.height; - params.bound_max_x = batch->key.width - 1; - params.bound_max_y = batch->key.height - 1; - params.effective_tile_size = tib_size; - params.tie_break_rule = MALI_TIE_BREAK_RULE_MINUS_180_IN_0_OUT; - params.render_target_count = rt_descriptors; - params.z_internal_format = get_z_internal_format(batch); - - if (batch->clear & PIPE_CLEAR_DEPTH) - params.z_clear = batch->clear_depth; - if (batch->clear & PIPE_CLEAR_STENCIL) - params.s_clear = batch->clear_stencil & 0xff; - - params.color_buffer_allocation = internal_cbuf_size; - - params.sample_count = util_framebuffer_get_num_samples(&batch->key); - params.sample_pattern = panfrost_sample_pattern(params.sample_count); - - if (batch->key.zsbuf && - ((batch->clear | batch->draws) & PIPE_CLEAR_DEPTHSTENCIL)) { - params.z_write_enable = true; - if (batch->key.zsbuf->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) - params.s_write_enable = true; - } - - params.has_zs_crc_extension = !!zs_crc_ext; - - if (checksum_slice) { - bool valid = checksum_slice->crc_valid; - bool full = !batch->minx && !batch->miny && - batch->maxx == batch->key.width && - batch->maxy == batch->key.height; - - params.crc_read_enable = valid; - - /* If the data is currently invalid, still write CRC - * data if we are doing a full write, so that it is - * valid for next time. */ - params.crc_write_enable = valid || full; - - checksum_slice->crc_valid |= full; - } - } - - if (pan_is_bifrost(dev)) - panfrost_mfbd_emit_bifrost_tiler(batch, fb, vertex_count); - else - panfrost_mfbd_emit_midgard_tiler(batch, fb, vertex_count); - - /* Return pointer suitable for the fragment section */ - unsigned tag = - MALI_FBD_TAG_IS_MFBD | - (zs_crc_ext ? MALI_FBD_TAG_HAS_ZS_RT : 0) | - (MALI_POSITIVE(rt_descriptors) << 2); - - return t.gpu | tag; -} diff --git a/src/gallium/drivers/panfrost/pan_screen.c b/src/gallium/drivers/panfrost/pan_screen.c index fdf044e7165..0ad365e602e 100644 --- a/src/gallium/drivers/panfrost/pan_screen.c +++ b/src/gallium/drivers/panfrost/pan_screen.c @@ -43,6 +43,7 @@ #include "drm-uapi/drm_fourcc.h" #include "drm-uapi/panfrost_drm.h" +#include "pan_blitter.h" #include "pan_bo.h" #include "pan_shader.h" #include "pan_screen.h" @@ -687,6 +688,7 @@ panfrost_destroy_screen(struct pipe_screen *pscreen) struct panfrost_device *dev = pan_device(pscreen); panfrost_cleanup_indirect_draw_shaders(dev); + pan_blitter_cleanup(dev); pan_blend_shaders_cleanup(dev); if (dev->ro) @@ -859,9 +861,9 @@ panfrost_create_screen(int fd, struct renderonly *ro) screen->base.set_damage_region = panfrost_resource_set_damage_region; panfrost_resource_screen_init(&screen->base); - panfrost_init_blit_shaders(dev); pan_blend_shaders_init(dev); panfrost_init_indirect_draw_shaders(dev); + pan_blitter_init(dev); return &screen->base; } diff --git a/src/gallium/drivers/panfrost/pan_sfbd.c b/src/gallium/drivers/panfrost/pan_sfbd.c deleted file mode 100644 index 29dd2f7480f..00000000000 --- a/src/gallium/drivers/panfrost/pan_sfbd.c +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright (C) 2019-2020 Collabora, Ltd. - * Copyright 2018-2019 Alyssa Rosenzweig - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include "pan_bo.h" -#include "pan_context.h" -#include "pan_util.h" - -#include "util/format/u_format.h" - -static void -panfrost_sfbd_format(struct pipe_surface *surf, - struct MALI_SINGLE_TARGET_FRAMEBUFFER_PARAMETERS *fb) -{ - /* Explode details on the format */ - - const struct util_format_description *desc = - util_format_description(surf->format); - - /* The swizzle for rendering is inverted from texturing */ - - unsigned char swizzle[4]; - panfrost_invert_swizzle(desc->swizzle, swizzle); - - fb->swizzle = panfrost_translate_swizzle_4(swizzle); - - struct pan_blendable_format fmt = panfrost_blend_format(surf->format); - - if (fmt.internal) { - fb->internal_format = fmt.internal; - fb->color_writeback_format = fmt.writeback; - } else { - unreachable("raw formats not finished for SFBD"); - } -} - -static void -panfrost_sfbd_clear( - struct panfrost_batch *batch, - struct MALI_SINGLE_TARGET_FRAMEBUFFER_PARAMETERS *sfbd) -{ - if (batch->clear & PIPE_CLEAR_COLOR) { - sfbd->clear_color_0 = batch->clear_color[0][0]; - sfbd->clear_color_1 = batch->clear_color[0][1]; - sfbd->clear_color_2 = batch->clear_color[0][2]; - sfbd->clear_color_3 = batch->clear_color[0][3]; - } - - if (batch->clear & PIPE_CLEAR_DEPTH) - sfbd->z_clear = batch->clear_depth; - - if (batch->clear & PIPE_CLEAR_STENCIL) - sfbd->s_clear = batch->clear_stencil & 0xff; -} - -static void -panfrost_sfbd_set_cbuf( - struct MALI_SINGLE_TARGET_FRAMEBUFFER_PARAMETERS *fb, - struct pipe_surface *surf) -{ - struct panfrost_resource *rsrc = pan_resource(surf->texture); - - unsigned level = surf->u.tex.level; - unsigned first_layer = surf->u.tex.first_layer; - assert(surf->u.tex.last_layer == first_layer); - signed row_stride = rsrc->image.layout.slices[level].row_stride; - - mali_ptr base = panfrost_get_texture_address(rsrc, level, first_layer, 0); - - panfrost_sfbd_format(surf, fb); - - fb->color_write_enable = true; - fb->color_writeback.base = base; - fb->color_writeback.row_stride = row_stride; - - if (rsrc->image.layout.modifier == DRM_FORMAT_MOD_LINEAR) - fb->color_block_format = MALI_BLOCK_FORMAT_LINEAR; - else if (rsrc->image.layout.modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { - fb->color_block_format = MALI_BLOCK_FORMAT_TILED_U_INTERLEAVED; - } else { - fprintf(stderr, "Invalid render modifier\n"); - assert(0); - } -} - -static void -panfrost_sfbd_set_zsbuf( - struct MALI_SINGLE_TARGET_FRAMEBUFFER_PARAMETERS *fb, - struct pipe_surface *surf) -{ - struct panfrost_resource *rsrc = pan_resource(surf->texture); - - unsigned level = surf->u.tex.level; - assert(surf->u.tex.first_layer == 0); - - fb->zs_writeback.base = rsrc->image.data.bo->ptr.gpu + - rsrc->image.layout.slices[level].offset; - fb->zs_writeback.row_stride = rsrc->image.layout.slices[level].row_stride; - - if (rsrc->image.layout.modifier == DRM_FORMAT_MOD_LINEAR) - fb->zs_block_format = MALI_BLOCK_FORMAT_LINEAR; - else if (rsrc->image.layout.modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { - fb->zs_block_format = MALI_BLOCK_FORMAT_TILED_U_INTERLEAVED; - } else { - fprintf(stderr, "Invalid render modifier\n"); - assert(0); - } - - switch (surf->format) { - case PIPE_FORMAT_Z16_UNORM: - fb->zs_format = MALI_ZS_FORMAT_D16; - break; - case PIPE_FORMAT_Z24_UNORM_S8_UINT: - fb->zs_format = MALI_ZS_FORMAT_D24S8; - break; - case PIPE_FORMAT_Z24X8_UNORM: - fb->zs_format = MALI_ZS_FORMAT_D24X8; - break; - case PIPE_FORMAT_Z32_FLOAT: - fb->zs_format = MALI_ZS_FORMAT_D32; - break; - case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: - fb->zs_format = MALI_ZS_FORMAT_D32_S8X24; - break; - default: - unreachable("Unsupported depth/stencil format."); - } -} - -static void -panfrost_init_sfbd_params(struct panfrost_batch *batch, - struct MALI_SINGLE_TARGET_FRAMEBUFFER_PARAMETERS *sfbd) -{ - sfbd->bound_max_x = batch->key.width - 1; - sfbd->bound_max_y = batch->key.height - 1; - sfbd->dithering_enable = true; - sfbd->clean_pixel_write_enable = true; - sfbd->tie_break_rule = MALI_TIE_BREAK_RULE_MINUS_180_IN_0_OUT; -} - -static void -panfrost_emit_sfdb_local_storage(struct panfrost_batch *batch, void *sfbd, - unsigned vertex_count) -{ - struct panfrost_device *dev = pan_device(batch->ctx->base.screen); - /* TODO: Why do we need to make the stack bigger than other platforms? */ - unsigned shift = panfrost_get_stack_shift(MAX2(batch->stack_size, 512)); - - pan_section_pack(sfbd, SINGLE_TARGET_FRAMEBUFFER, LOCAL_STORAGE, ls) { - ls.tls_size = shift; - ls.wls_instances = MALI_LOCAL_STORAGE_NO_WORKGROUP_MEM; - ls.tls_base_pointer = - panfrost_batch_get_scratchpad(batch, - shift, - dev->thread_tls_alloc, - dev->core_count)->ptr.gpu; - } -} - -static void -panfrost_emit_sfdb_tiler(struct panfrost_batch *batch, void *sfbd, - unsigned vertex_count) -{ - void *tiler = pan_section_ptr(sfbd, SINGLE_TARGET_FRAMEBUFFER, TILER); - - panfrost_emit_midg_tiler(batch, tiler, vertex_count); - - /* All weights set to 0, nothing to do here */ - pan_section_pack(sfbd, SINGLE_TARGET_FRAMEBUFFER, PADDING_1, padding) {} - pan_section_pack(sfbd, SINGLE_TARGET_FRAMEBUFFER, TILER_WEIGHTS, w) {} -} - -void -panfrost_attach_sfbd(struct panfrost_batch *batch, unsigned vertex_count) -{ - void *sfbd = batch->framebuffer.cpu; - - panfrost_emit_sfdb_local_storage(batch, sfbd, vertex_count); - pan_section_pack(sfbd, SINGLE_TARGET_FRAMEBUFFER, PARAMETERS, params) { - panfrost_init_sfbd_params(batch, ¶ms); - } - panfrost_emit_sfdb_tiler(batch, sfbd, vertex_count); - pan_section_pack(sfbd, SINGLE_TARGET_FRAMEBUFFER, PADDING_2, padding) {} -} - -/* Creates an SFBD for the FRAGMENT section of the bound framebuffer */ - -mali_ptr -panfrost_sfbd_fragment(struct panfrost_batch *batch, bool has_draws) -{ - struct panfrost_ptr t = - panfrost_pool_alloc_desc(&batch->pool, SINGLE_TARGET_FRAMEBUFFER); - void *sfbd = t.cpu; - - panfrost_emit_sfdb_local_storage(batch, sfbd, has_draws); - pan_section_pack(sfbd, SINGLE_TARGET_FRAMEBUFFER, PARAMETERS, params) { - panfrost_init_sfbd_params(batch, ¶ms); - panfrost_sfbd_clear(batch, ¶ms); - - /* SFBD does not support MRT natively; sanity check */ - assert(batch->key.nr_cbufs <= 1); - if (batch->key.nr_cbufs && batch->key.cbufs[0]) { - struct pipe_surface *surf = batch->key.cbufs[0]; - struct panfrost_resource *rsrc = pan_resource(surf->texture); - - panfrost_sfbd_set_cbuf(¶ms, surf); - - if (rsrc->image.layout.crc_mode != PAN_IMAGE_CRC_NONE) { - unsigned level = surf->u.tex.level; - struct pan_image_slice_layout *slice = &rsrc->image.layout.slices[level]; - - if (rsrc->image.layout.crc_mode == PAN_IMAGE_CRC_INBAND) { - params.crc_buffer.base = rsrc->image.data.bo->ptr.gpu + - rsrc->image.data.offset + - slice->crc.offset; - } else { - params.crc_buffer.base = rsrc->image.crc.bo->ptr.gpu + - slice->crc.offset; - } - params.crc_buffer.row_stride = slice->crc.stride; - } - } - - if (batch->key.zsbuf) - panfrost_sfbd_set_zsbuf(¶ms, batch->key.zsbuf); - - params.sample_count = util_framebuffer_get_num_samples(&batch->key); - - /* XXX: different behaviour from MFBD and probably wrong... */ - params.msaa = (params.sample_count > 1) ? - MALI_MSAA_MULTIPLE : - MALI_MSAA_SINGLE; - } - panfrost_emit_sfdb_tiler(batch, sfbd, has_draws); - pan_section_pack(sfbd, SINGLE_TARGET_FRAMEBUFFER, PADDING_2, padding) {} - - return t.gpu; -}